From b7c652ab07562f3beb40239901cb6b5a3f33f09b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 9 Mar 2020 18:23:44 +0200 Subject: [PATCH 001/102] Refactoring of Queue Interfaces --- .../thingsboard/server/TbQueueCallback.java | 8 ++++ .../thingsboard/server/TbQueueConsumer.java | 11 +++++ .../org/thingsboard/server/TbQueueMsg.java | 13 ++++++ .../server/TbQueueMsgMetadata.java | 4 ++ .../thingsboard/server/TbQueueProducer.java | 9 ++++ .../server/TbQueueRequestTemplate.java | 9 ++++ .../transport/queue/TransportApiCall.java | 31 +++++++++++++ .../queue/TransportApiCallRequest.java | 40 +++++++++++++++++ .../queue/TransportQueueProvider.java | 19 ++++++++ .../service/AbstractTransportService.java | 5 +++ .../src/main/resources/tb-mqtt-transport.yml | 43 +++++++++---------- 11 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java new file mode 100644 index 0000000000..a1b64bd205 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueCallback { + + void onSuccess(); + + void onFailure(Throwable t); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java new file mode 100644 index 0000000000..02010522e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -0,0 +1,11 @@ +package org.thingsboard.server; + +import java.util.List; + +public interface TbQueueConsumer { + + List poll(); + + void commit(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java new file mode 100644 index 0000000000..4f0e5b7f2e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -0,0 +1,13 @@ +package org.thingsboard.server; + +import java.util.Map; +import java.util.UUID; + +public interface TbQueueMsg { + + UUID getKey(); + + Map getHeaders(); + + byte[] getData(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java new file mode 100644 index 0000000000..8eecae51fb --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -0,0 +1,4 @@ +package org.thingsboard.server; + +public interface TbQueueMsgMetadata { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java new file mode 100644 index 0000000000..a36deda8e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueProducer { + + ListenableFuture send(T msg, TbQueueCallback callback); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java new file mode 100644 index 0000000000..a820d5fcb4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueRequestTemplate { + + ListenableFuture post(String key, Request request); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java new file mode 100644 index 0000000000..956c1138f1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java @@ -0,0 +1,31 @@ +package org.thingsboard.server.common.transport.queue; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public abstract class TransportApiCall { + + protected byte[] uuidToBytes(UUID uuid) { + ByteBuffer buf = ByteBuffer.allocate(16); + buf.putLong(uuid.getMostSignificantBits()); + buf.putLong(uuid.getLeastSignificantBits()); + return buf.array(); + } + + protected static UUID bytesToUuid(byte[] bytes) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + long firstLong = bb.getLong(); + long secondLong = bb.getLong(); + return new UUID(firstLong, secondLong); + } + + protected byte[] stringToBytes(String string) { + return string.getBytes(StandardCharsets.UTF_8); + } + + protected String bytesToString(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java new file mode 100644 index 0000000000..49b83e2c17 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java @@ -0,0 +1,40 @@ +package org.thingsboard.server.common.transport.queue; + +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class TransportApiCallRequest extends TransportApiCall implements TbQueueMsg { + public static final String REQUEST_ID_HEADER = "requestId"; + public static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + private final UUID requestId; + private final Map headers; + private final TransportProtos.TransportApiRequestMsg msg; + + public TransportApiCallRequest(UUID requestId, String responseTopic, TransportProtos.TransportApiRequestMsg msg) { + this.requestId = requestId; + this.headers = new HashMap<>(); + this.headers.put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + this.headers.put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTopic)); + this.msg = msg; + } + + @Override + public UUID getKey() { + return requestId; + } + + @Override + public Map getHeaders() { + return null; + } + + @Override + public byte[] getData() { + return msg.toByteArray(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java new file mode 100644 index 0000000000..d4ce11afec --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java @@ -0,0 +1,19 @@ +package org.thingsboard.server.common.transport.queue; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueProducer; + +public interface TransportQueueProvider { + + TbQueueProducer getTransportApiCallRequestsProducer(); + + TbQueueConsumer getTransportApiCallResponsesConsumer(); + + TbQueueProducer getRuleEngineMsgProducer(); + + TbQueueProducer getTbCoreMsgProducer(); + + TbQueueConsumer getTransportNotificationsConsumer(); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java index 77c45e1714..0aaa8cef68 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; @@ -26,6 +27,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.queue.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Random; @@ -49,6 +51,9 @@ public abstract class AbstractTransportService implements TransportService { @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Autowired + private TransportQueueProvider queueProvider; + protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index d13a3d071e..47dfe57c85 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -61,25 +61,24 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + queue: + type: kafka + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" From a39e8c37567ba8a1e07d78dced5a22f876c4d644 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 12:34:05 +0200 Subject: [PATCH 002/102] Queue Implementation --- .../org/thingsboard/server/TbQueueAdmin.java | 9 + .../thingsboard/server/TbQueueCallback.java | 2 +- .../thingsboard/server/TbQueueConsumer.java | 6 +- .../thingsboard/server/TbQueueHandler.java | 27 +++ .../org/thingsboard/server/TbQueueMsg.java | 3 +- .../thingsboard/server/TbQueueMsgHeaders.java | 8 + .../thingsboard/server/TbQueueProducer.java | 6 + .../server/TbQueueRequestTemplate.java | 2 +- .../server/TbQueueResponseTemplate.java | 8 + .../common/AbstractTbQueueTemplate.java | 32 ++++ .../server/common/AsyncCallbackTemplate.java | 66 +++++++ .../common/DefaultTbQueueRequestTemplate.java | 172 ++++++++++++++++++ .../DefaultTbQueueResponseTemplate.java | 137 ++++++++++++++ 13 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java new file mode 100644 index 0000000000..7f8cff5f22 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueAdmin { + + ListenableFuture createTopicIfNotExists(String topic); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index a1b64bd205..3d7d791ae4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -2,7 +2,7 @@ package org.thingsboard.server; public interface TbQueueCallback { - void onSuccess(); + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index 02010522e4..181d50bdd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -4,7 +4,11 @@ import java.util.List; public interface TbQueueConsumer { - List poll(); + String getTopic(); + + void subscribe(); + + List poll(long durationInMillis); void commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java new file mode 100644 index 0000000000..93004cddca --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Created by ashvayka on 05.10.18. + */ +public interface TbQueueHandler { + + ListenableFuture handle(Request request); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 4f0e5b7f2e..22714af77e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,13 +1,12 @@ package org.thingsboard.server; -import java.util.Map; import java.util.UUID; public interface TbQueueMsg { UUID getKey(); - Map getHeaders(); + TbQueueMsgHeaders getHeaders(); byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java new file mode 100644 index 0000000000..e1a5b4861d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueMsgHeaders { + + byte[] put(String key, byte[] value); + + byte[] get(String key); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index a36deda8e4..41d165d743 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -4,6 +4,12 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueProducer { + void init(); + + String getDefaultTopic(); + ListenableFuture send(T msg, TbQueueCallback callback); + ListenableFuture send(String topic, T msg, TbQueueCallback callback); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index a820d5fcb4..85b5cc3f15 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -4,6 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { - ListenableFuture post(String key, Request request); + ListenableFuture send(String key, Request request); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java new file mode 100644 index 0000000000..f3b4e4ad5d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueResponseTemplate { + + void init(TbQueueHandler handler); + + void stop(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java new file mode 100644 index 0000000000..2f406a9631 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -0,0 +1,32 @@ +package org.thingsboard.server.common; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class AbstractTbQueueTemplate { + protected static final String REQUEST_ID_HEADER = "requestId"; + protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + protected byte[] uuidToBytes(UUID uuid) { + ByteBuffer buf = ByteBuffer.allocate(16); + buf.putLong(uuid.getMostSignificantBits()); + buf.putLong(uuid.getLeastSignificantBits()); + return buf.array(); + } + + protected static UUID bytesToUuid(byte[] bytes) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + long firstLong = bb.getLong(); + long secondLong = bb.getLong(); + return new UUID(firstLong, secondLong); + } + + protected byte[] stringToBytes(String string) { + return string.getBytes(StandardCharsets.UTF_8); + } + + protected String bytesToString(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java new file mode 100644 index 0000000000..95783f87cd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 05.10.18. + */ +public class AsyncCallbackTemplate { + + public static void withCallbackAndTimeout(ListenableFuture future, + Consumer onSuccess, + Consumer onFailure, + long timeoutInMs, + ScheduledExecutorService timeoutExecutor, + Executor callbackExecutor) { + future = Futures.withTimeout(future, timeoutInMs, TimeUnit.MILLISECONDS, timeoutExecutor); + withCallback(future, onSuccess, onFailure, callbackExecutor); + } + + public static void withCallback(ListenableFuture future, Consumer onSuccess, + Consumer onFailure, Executor executor) { + FutureCallback callback = new FutureCallback() { + @Override + public void onSuccess(T result) { + try { + onSuccess.accept(result); + } catch (Throwable th) { + onFailure(th); + } + } + + @Override + public void onFailure(Throwable t) { + onFailure.accept(t); + } + }; + if (executor != null) { + Futures.addCallback(future, callback, executor); + } else { + Futures.addCallback(future, callback); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java new file mode 100644 index 0000000000..2965b7a291 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -0,0 +1,172 @@ +package org.thingsboard.server.common; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.TbQueueAdmin; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate + implements TbQueueRequestTemplate { + + private final TbQueueAdmin queueAdmin; + private final TbQueueProducer requestTemplate; + private final TbQueueConsumer responseTemplate; + private final ConcurrentMap> pendingRequests; + private final boolean internalExecutor; + private final ExecutorService executor; + private final long maxRequestTimeout; + private final long maxPendingRequests; + private final long pollInterval; + private volatile long tickTs = 0L; + private volatile long tickSize = 0L; + private volatile boolean stopped = false; + + @Builder + public DefaultTbQueueRequestTemplate(TbQueueAdmin queueAdmin, + TbQueueProducer requestTemplate, + TbQueueConsumer responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { + this.queueAdmin = queueAdmin; + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxRequestTimeout = maxRequestTimeout; + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + if (executor != null) { + internalExecutor = false; + this.executor = executor; + } else { + internalExecutor = true; + this.executor = Executors.newSingleThreadExecutor(); + } + } + + public void init() { + queueAdmin.createTopicIfNotExists(responseTemplate.getTopic()); + this.requestTemplate.init(); + tickTs = System.currentTimeMillis(); + responseTemplate.subscribe(); + executor.submit(() -> { + long nextCleanupMs = 0L; + while (!stopped) { + try { + List responses = responseTemplate.poll(pollInterval); + if (responses.size() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } + responses.forEach(response -> { + log.trace("Received response to Kafka Template request: {}", response); + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId = null; + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header and body", response); + } else { + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received", requestId); + ResponseMetaData expectedResponse = pendingRequests.remove(requestId); + if (expectedResponse == null) { + log.trace("[{}] Invalid or stale request", requestId); + } else { + expectedResponse.future.set(response); + } + } + }); + responseTemplate.commit(); + tickTs = System.currentTimeMillis(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + staleRequest.future.setException(new TimeoutException()); + } + } + }); + nextCleanupMs = tickTs + maxRequestTimeout; + } + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from kafka was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain responses from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new responses", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (internalExecutor) { + executor.shutdownNow(); + } + } + + @Override + public ListenableFuture send(String key, Request request) { + if (tickSize > maxPendingRequests) { + return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); + } + UUID requestId = UUID.randomUUID(); + request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); + SettableFuture future = SettableFuture.create(); + ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); + pendingRequests.putIfAbsent(requestId, responseMetaData); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + requestTemplate.send(request, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}] Request sent: {}", requestId, metadata); + } + + @Override + public void onFailure(Throwable t) { + pendingRequests.remove(requestId); + future.setException(t); + } + }); + return future; + } + + private static class ResponseMetaData { + private final long expTime; + private final SettableFuture future; + + ResponseMetaData(long ts, SettableFuture future) { + this.expTime = ts; + this.future = future; + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java new file mode 100644 index 0000000000..92dac0551b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -0,0 +1,137 @@ +package org.thingsboard.server.common; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueHandler; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueResponseTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate + implements TbQueueResponseTemplate { + + private final TbQueueConsumer requestTemplate; + private final TbQueueProducer responseTemplate; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, + TbQueueProducer responseTemplate, + TbQueueHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + public void init(TbQueueHandler handler) { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + List requests = requestTemplate.poll(pollInterval); + requests.forEach(request -> { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(responseTopic, response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + }); + requestTemplate.commit(); + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from queue was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + +} From 7b2bdeab236f75b16f246c844665eaeea7162820 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 14:03:06 +0200 Subject: [PATCH 003/102] Refactoring of Transport Interfaces --- common/queue/pom.xml | 4 + .../server/TbQueueRequestTemplate.java | 2 +- .../common/DefaultTbQueueMsgHeaders.java | 21 ++ .../common/DefaultTbQueueRequestTemplate.java | 4 +- .../server/common/TbProtoQueueMsg.java | 40 ++++ .../queue/KafkaTransportQueueProvider.java | 31 +++ .../transport/queue/TransportApiCall.java | 31 --- .../queue/TransportApiCallRequest.java | 40 ---- .../queue/TransportQueueProvider.java | 17 +- .../service/AbstractTransportService.java | 27 ++- .../service/RemoteTransportService.java | 200 +++++++++--------- 11 files changed, 234 insertions(+), 183 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 8f0a0913a4..952b3c540d 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -94,6 +94,10 @@ mockito-all test + + com.google.protobuf + protobuf-java + diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 85b5cc3f15..5ba28f22c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -4,6 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { - ListenableFuture send(String key, Request request); + ListenableFuture send(Request request); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java new file mode 100644 index 0000000000..4ef33a489d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -0,0 +1,21 @@ +package org.thingsboard.server.common; + +import org.thingsboard.server.TbQueueMsgHeaders; + +import java.util.HashMap; +import java.util.Map; + +public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { + + private final Map data = new HashMap<>(); + + @Override + public byte[] put(String key, byte[] value) { + return data.put(key, value); + } + + @Override + public byte[] get(String key) { + return data.get(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 2965b7a291..49022ad2e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -133,7 +133,7 @@ public class DefaultTbQueueRequestTemplate send(String key, Request request) { + public ListenableFuture send(Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } @@ -143,7 +143,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); requestTemplate.send(request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java new file mode 100644 index 0000000000..f1120b876f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -0,0 +1,40 @@ +package org.thingsboard.server.common; + +import lombok.Data; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbProtoQueueMsg implements TbQueueMsg { + + private final UUID key; + private final T value; + private final DefaultTbQueueMsgHeaders headers; + + public TbProtoQueueMsg(UUID key, T value) { + this(key, value, new DefaultTbQueueMsgHeaders()); + } + + public TbProtoQueueMsg(UUID key, T value, DefaultTbQueueMsgHeaders headers) { + this.key = key; + this.value = value; + this.headers = headers; + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return value.toByteArray(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java new file mode 100644 index 0000000000..2aa6f90b7b --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java @@ -0,0 +1,31 @@ +package org.thingsboard.server.common.transport.queue; + +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +@Component +public class KafkaTransportQueueProvider implements TransportQueueProvider { + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return null; + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java deleted file mode 100644 index 956c1138f1..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -public abstract class TransportApiCall { - - protected byte[] uuidToBytes(UUID uuid) { - ByteBuffer buf = ByteBuffer.allocate(16); - buf.putLong(uuid.getMostSignificantBits()); - buf.putLong(uuid.getLeastSignificantBits()); - return buf.array(); - } - - protected static UUID bytesToUuid(byte[] bytes) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - return new UUID(firstLong, secondLong); - } - - protected byte[] stringToBytes(String string) { - return string.getBytes(StandardCharsets.UTF_8); - } - - protected String bytesToString(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } - -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java deleted file mode 100644 index 49b83e2c17..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class TransportApiCallRequest extends TransportApiCall implements TbQueueMsg { - public static final String REQUEST_ID_HEADER = "requestId"; - public static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - - private final UUID requestId; - private final Map headers; - private final TransportProtos.TransportApiRequestMsg msg; - - public TransportApiCallRequest(UUID requestId, String responseTopic, TransportProtos.TransportApiRequestMsg msg) { - this.requestId = requestId; - this.headers = new HashMap<>(); - this.headers.put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - this.headers.put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTopic)); - this.msg = msg; - } - - @Override - public UUID getKey() { - return requestId; - } - - @Override - public Map getHeaders() { - return null; - } - - @Override - public byte[] getData() { - return msg.toByteArray(); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java index d4ce11afec..d0bbafc615 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java @@ -1,19 +1,22 @@ package org.thingsboard.server.common.transport.queue; import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; public interface TransportQueueProvider { - TbQueueProducer getTransportApiCallRequestsProducer(); + TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); - TbQueueConsumer getTransportApiCallResponsesConsumer(); + TbQueueProducer> getRuleEngineMsgProducer(); - TbQueueProducer getRuleEngineMsgProducer(); + TbQueueProducer> getTbCoreMsgProducer(); - TbQueueProducer getTbCoreMsgProducer(); - - TbQueueConsumer getTransportNotificationsConsumer(); + TbQueueConsumer> getTransportNotificationsConsumer(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java index 0aaa8cef68..1a66ea9601 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -29,6 +33,10 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.queue.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import java.util.Random; import java.util.UUID; @@ -54,6 +62,15 @@ public abstract class AbstractTransportService implements TransportService { @Autowired private TransportQueueProvider queueProvider; + + protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; + + protected TbQueueProducer> ruleEngineMsgProducer; + + protected TbQueueProducer> tbCoreMsgProducer; + + protected TbQueueConsumer> transportNotificationsConsumer; + protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; @@ -296,8 +313,8 @@ public abstract class AbstractTransportService implements TransportService { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } - protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); + protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } public void init() { @@ -309,6 +326,10 @@ public abstract class AbstractTransportService implements TransportService { this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); } public void destroy() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java index 17031791d5..58685db764 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,16 +16,14 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; @@ -43,22 +41,18 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.time.Duration; +import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -97,10 +91,6 @@ public class RemoteTransportService extends AbstractTransportService { @Autowired private TbNodeIdProvider nodeIdProvider; - private TbKafkaRequestTemplate transportApiTemplate; - private TBKafkaProducerTemplate ruleEngineProducer; - private TBKafkaConsumerTemplate mainConsumer; - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); private volatile boolean stopped = false; @@ -109,67 +99,67 @@ public class RemoteTransportService extends AbstractTransportService { public void init() { super.init(); - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiRequestsTopic); - requestBuilder.encoder(new TransportApiRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-api-client"); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new TransportApiResponseDecoder()); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - transportApiTemplate = builder.build(); - transportApiTemplate.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); - ruleEngineProducerBuilder.settings(kafkaSettings); - ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); - ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); - ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); - ruleEngineProducer = ruleEngineProducerBuilder.build(); - ruleEngineProducer.init(); - - String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); - - try { - TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); - CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - log.trace("Failed to create topic: {}", e.getMessage(), e); - } - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); - mainConsumerBuilder.settings(kafkaSettings); - mainConsumerBuilder.topic(notificationsTopicName); - mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - mainConsumerBuilder.groupId("transport"); - mainConsumerBuilder.autoCommit(true); - mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); - mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); - mainConsumer = mainConsumerBuilder.build(); - mainConsumer.subscribe(); +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); +// requestBuilder.settings(kafkaSettings); +// requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); +// requestBuilder.defaultTopic(transportApiRequestsTopic); +// requestBuilder.encoder(new TransportApiRequestEncoder()); +// +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); +// responseBuilder.settings(kafkaSettings); +// responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); +// responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); +// responseBuilder.groupId("transport-api-client"); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); +// +// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder +// builder = TbKafkaRequestTemplate.builder(); +// builder.requestTemplate(requestBuilder.build()); +// builder.responseTemplate(responseBuilder.build()); +// builder.maxPendingRequests(maxPendingRequests); +// builder.maxRequestTimeout(maxRequestsTimeout); +// builder.pollInterval(responsePollDuration); +// transportApiTemplate = builder.build(); +// transportApiTemplate.init(); +// +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); +// ruleEngineProducerBuilder.settings(kafkaSettings); +// ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); +// ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); +// ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); +// ruleEngineProducer = ruleEngineProducerBuilder.build(); +// ruleEngineProducer.init(); +// +// String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); +// +// try { +// TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); +// CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); +// result.all().get(); +// } catch (Exception e) { +// log.trace("Failed to create topic: {}", e.getMessage(), e); +// } +// +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); +// mainConsumerBuilder.settings(kafkaSettings); +// mainConsumerBuilder.topic(notificationsTopicName); +// mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); +// mainConsumerBuilder.groupId("transport"); +// mainConsumerBuilder.autoCommit(true); +// mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); +// mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); +// mainConsumer = mainConsumerBuilder.build(); +// mainConsumer.subscribe(); mainConsumerExecutor.execute(() -> { while (!stopped) { try { - ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); records.forEach(record -> { try { - ToTransportMsg toTransportMsg = mainConsumer.decode(record); + ToTransportMsg toTransportMsg = record.getValue(); if (toTransportMsg.hasToDeviceSessionMsg()) { processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); } @@ -193,12 +183,12 @@ public class RemoteTransportService extends AbstractTransportService { public void destroy() { super.destroy(); stopped = true; - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } +// if (transportApiTemplate != null) { +// transportApiTemplate.stop(); +// } +// if (mainConsumer != null) { +// mainConsumer.unsubscribe(); +// } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } @@ -207,25 +197,25 @@ public class RemoteTransportService extends AbstractTransportService { @Override public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), - TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); } @Override public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), - TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); } @Override public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), - TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); } @Override @@ -322,18 +312,30 @@ public class RemoteTransportService extends AbstractTransportService { } private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { - if (callback != null) { - if (exception == null) { - this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } else { - this.transportCallbackExecutor.submit(() -> { - callback.onError(exception); - }); - } - } - }); + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + RemoteTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); + } + + @Override + public void onFailure(Throwable t) { + RemoteTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onError(t); + }); + } } } From f8cc5b3cd2598b47207330e78a4bd23184f74fa6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 17:47:16 +0200 Subject: [PATCH 004/102] Transport service refactoring --- .../transport/LocalTransportService.java | 227 ------------ .../transport/mqtt/MqttTransportHandler.java | 8 +- .../mqtt/session/GatewaySessionHandler.java | 6 +- ...vice.java => DefaultTransportService.java} | 247 ++++++++++--- .../service/RemoteTransportService.java | 341 ------------------ 5 files changed, 194 insertions(+), 635 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java rename common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/{AbstractTransportService.java => DefaultTransportService.java} (62%) delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java deleted file mode 100644 index 32ff77877d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.service.AbstractTransportService; -import org.thingsboard.server.dao.device.ClaimDevicesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") -public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { - - @Autowired - private TransportApiService transportApiService; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - @Autowired - private ClaimDevicesService claimDevicesService; - - @PostConstruct - public void init() { - super.init(); - } - - @PreDestroy - public void destroy() { - super.destroy(); - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); - } - - @Override - public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); - - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - callback.onSuccess(null); - } else { - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); - DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), - callback::onSuccess, callback::onError); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - processToTransportMsg(msg); - if (onSuccess != null) { - onSuccess.run(); - } - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - if (callback != null) { - callback.onSuccess(null); - } - } - - private Consumer getThrowableConsumer(TransportServiceCallback callback) { - return e -> { - if (callback != null) { - callback.onError(e); - } - }; - } - -} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 84c7598d9b..0d96f94252 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -411,7 +411,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement ctx.close(); log.info("[{}] Client disconnected!", sessionId); if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); if (gatewaySessionHandler != null) { gatewaySessionHandler.onGatewayDisconnect(); @@ -486,7 +486,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); } } @@ -506,7 +506,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) .build(); - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index e0a944ed60..de5f82f324 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java similarity index 62% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 1a66ea9601..dd7d9bd27a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,8 +18,11 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; @@ -37,16 +40,27 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; import java.util.Random; import java.util.UUID; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * Created by ashvayka on 17.10.18. */ @Slf4j -public abstract class AbstractTransportService implements TransportService { +@Service +public class DefaultTransportService implements TransportService { @Value("${transport.rate_limits.enabled}") private boolean rateLimitEnabled; @@ -62,34 +76,134 @@ public abstract class AbstractTransportService implements TransportService { @Autowired private TransportQueueProvider queueProvider; + @Value("${kafka.notifications.poll_interval}") + private int notificationsPollDuration; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; - protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; - protected TbQueueConsumer> transportNotificationsConsumer; protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; private ConcurrentMap sessions = new ConcurrentHashMap<>(); - //TODO: Implement cleanup of this maps. private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); + private volatile boolean stopped = false; + + @PostConstruct + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + records.forEach(record -> { + try { + ToTransportMsg toTransportMsg = record.getValue(); + if (toTransportMsg.hasToDeviceSessionMsg()) { + processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @PreDestroy + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + stopped = true; + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } + @Override + public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -97,7 +211,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostTelemetry(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -105,7 +223,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -113,7 +235,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -122,7 +248,11 @@ public abstract class AbstractTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -131,7 +261,11 @@ public abstract class AbstractTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -139,7 +273,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -147,14 +285,22 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToServerRPCCallRequest(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - registerClaimingInfo(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } @Override @@ -162,24 +308,6 @@ public abstract class AbstractTransportService implements TransportService { reportActivityInternal(sessionInfo); } - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); - - protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { UUID sessionId = toId(sessionInfo); SessionMetaData sessionMetaData = sessions.get(sessionId); @@ -317,37 +445,36 @@ public abstract class AbstractTransportService implements TransportService { return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); - ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); } - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); + protected void send(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); } - } - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); + @Override + public void onFailure(Throwable t) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onError(t); + }); + } } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java deleted file mode 100644 index 58685db764..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Created by ashvayka on 05.10.18. - */ -@ConditionalOnExpression("'${transport.type:null}'=='null'") -@Service -@Slf4j -public class RemoteTransportService extends AbstractTransportService { - - @Value("${kafka.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${kafka.notifications.topic}") - private String notificationsTopic; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - @Value("${kafka.notifications.auto_commit_interval}") - private int notificationsAutoCommitInterval; - @Value("${kafka.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${kafka.transport_api.responses_topic}") - private String transportApiResponsesTopic; - @Value("${kafka.transport_api.max_pending_requests}") - private long maxPendingRequests; - @Value("${kafka.transport_api.max_requests_timeout}") - private long maxRequestsTimeout; - @Value("${kafka.transport_api.response_poll_interval}") - private int responsePollDuration; - @Value("${kafka.transport_api.response_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - //We use this to get the node id. We should replace this with a component that provides the node id. - @Autowired - private TbNodeIdProvider nodeIdProvider; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); - - private volatile boolean stopped = false; - - @PostConstruct - public void init() { - super.init(); - -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); -// requestBuilder.settings(kafkaSettings); -// requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); -// requestBuilder.defaultTopic(transportApiRequestsTopic); -// requestBuilder.encoder(new TransportApiRequestEncoder()); -// -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); -// responseBuilder.settings(kafkaSettings); -// responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); -// responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); -// responseBuilder.groupId("transport-api-client"); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); -// -// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder -// builder = TbKafkaRequestTemplate.builder(); -// builder.requestTemplate(requestBuilder.build()); -// builder.responseTemplate(responseBuilder.build()); -// builder.maxPendingRequests(maxPendingRequests); -// builder.maxRequestTimeout(maxRequestsTimeout); -// builder.pollInterval(responsePollDuration); -// transportApiTemplate = builder.build(); -// transportApiTemplate.init(); -// -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); -// ruleEngineProducerBuilder.settings(kafkaSettings); -// ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); -// ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); -// ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); -// ruleEngineProducer = ruleEngineProducerBuilder.build(); -// ruleEngineProducer.init(); -// -// String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); -// -// try { -// TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); -// CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); -// result.all().get(); -// } catch (Exception e) { -// log.trace("Failed to create topic: {}", e.getMessage(), e); -// } -// -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); -// mainConsumerBuilder.settings(kafkaSettings); -// mainConsumerBuilder.topic(notificationsTopicName); -// mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); -// mainConsumerBuilder.groupId("transport"); -// mainConsumerBuilder.autoCommit(true); -// mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); -// mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); -// mainConsumer = mainConsumerBuilder.build(); -// mainConsumer.subscribe(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> records = transportNotificationsConsumer.poll(notificationsPollDuration); - records.forEach(record -> { - try { - ToTransportMsg toTransportMsg = record.getValue(); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @PreDestroy - public void destroy() { - super.destroy(); - stopped = true; -// if (transportApiTemplate != null) { -// transportApiTemplate.stop(); -// } -// if (mainConsumer != null) { -// mainConsumer.unsubscribe(); -// } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? - new TransportTbQueueCallback(callback) : null); - - } - - private class TransportTbQueueCallback implements TbQueueCallback { - private final TransportServiceCallback callback; - - private TransportTbQueueCallback(TransportServiceCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - RemoteTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } - - @Override - public void onFailure(Throwable t) { - RemoteTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onError(t); - }); - } - } -} From 4ad4fe11daec122d021a96def79b16d3197c460c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 11 Mar 2020 17:18:35 +0200 Subject: [PATCH 005/102] created in memory queue --- .../device/DeviceActorMessageProcessor.java | 4 +- .../service/script/RemoteJsInvokeService.java | 86 +++++++-------- .../script/RemoteJsRequestEncoder.java | 7 +- .../script/RemoteJsResponseDecoder.java | 10 +- .../transport/LocalTransportApiService.java | 25 ++--- .../RemoteRuleEngineTransportService.java | 56 +++++----- .../transport/RemoteTransportApiService.java | 28 +++-- .../transport/ToRuleEngineMsgDecoder.java | 6 +- .../transport/TransportApiService.java | 5 +- .../thingsboard/server/TbQueueConsumer.java | 2 + .../thingsboard/server/TbQueueMsgHeaders.java | 4 + .../common/DefaultTbQueueMsgHeaders.java | 7 +- .../server/common/TbProtoQueueMsg.java | 4 +- .../server/kafka/KafkaTbQueueMsg.java | 39 +++++++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 12 ++ .../server/kafka/TBKafkaAdmin.java | 9 +- .../server/kafka/TBKafkaConsumerTemplate.java | 48 +++++++- .../server/kafka/TBKafkaProducerTemplate.java | 104 +++++++++++------- .../server/kafka/TbKafkaDecoder.java | 4 +- .../server/memory/InMemoryStorage.java | 45 ++++++++ .../memory/InMemoryTbQueueConsumer.java | 42 +++++++ .../memory/InMemoryTbQueueProducer.java | 43 ++++++++ .../service/DefaultTransportService.java | 2 +- .../ToTransportMsgResponseDecoder.java | 6 +- .../service/TransportApiResponseDecoder.java | 6 +- .../server/dao/entity/BaseEntityService.java | 13 ++- .../dao/relation/BaseRelationService.java | 38 ++++--- .../timescale/TimescaleTimeseriesDao.java | 3 +- .../CassandraBaseTimeseriesDao.java | 5 +- 29 files changed, 481 insertions(+), 182 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index acf1a3161c..2752b53090 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -20,9 +20,9 @@ import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -292,7 +292,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .build(); sendToTransport(responseMsg, sessionInfo); } - }); + }, MoreExecutors.directExecutor()); } private ListenableFuture>> getAttributesKvEntries(GetAttributeRequestMsg request) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 45d773f387..d39465ff12 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -25,11 +25,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; @@ -40,7 +39,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) @@ -99,42 +97,43 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private TbKafkaRequestTemplate kafkaTemplate; + private DefaultTbQueueRequestTemplate, TbProtoQueueMsg> defaultTemplate; private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(requestTopic); - requestBuilder.encoder(new RemoteJsRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new RemoteJsResponseDecoder()); - responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - kafkaTemplate = builder.build(); - kafkaTemplate.init(); +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); +// requestBuilder.settings(kafkaSettings); +// requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); +// requestBuilder.defaultTopic(requestTopic); +// requestBuilder.encoder(new RemoteJsRequestEncoder()); + TbQueueProducer> producer; + +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); +// responseBuilder.settings(kafkaSettings); +// responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); +// responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); +// responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new RemoteJsResponseDecoder()); +// responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); +// +// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder +// builder = TbKafkaRequestTemplate.builder(); +// builder.requestTemplate(requestBuilder.build()); +// builder.responseTemplate(responseBuilder.build()); +// builder.maxPendingRequests(maxPendingRequests); +// builder.maxRequestTimeout(maxRequestsTimeout); +// builder.pollInterval(responsePollDuration); +// defaultTemplate = builder.build(); +// defaultTemplate.init(); } @PreDestroy public void destroy() { - if (kafkaTemplate != null) { - kafkaTemplate.stop(); + if (defaultTemplate != null) { + defaultTemplate.stop(); } } @@ -151,11 +150,12 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaEvalMsgs.incrementAndGet(); } @@ -168,7 +168,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -202,11 +202,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaInvokeMsgs.incrementAndGet(); } @@ -219,7 +219,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { @@ -240,8 +240,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); - JsInvokeProtos.RemoteJsResponse response = future.get(); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index d07a2490ba..1b33fd85cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.kafka.TbKafkaEncoder; @@ -25,11 +26,11 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder { +public class RemoteJsRequestEncoder implements TbKafkaEncoder> { @Override - public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { + public byte[] encode(TbProtoQueueMsg value) { try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 7a3876fb91..ef15d5c877 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -25,12 +27,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder { +public class RemoteJsResponseDecoder implements TbKafkaDecoder> { @Override - public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { + public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); - return builder.build(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java index 37ff8ed93d..f6434036fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java @@ -21,10 +21,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -42,19 +41,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaResponseTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.locks.ReentrantLock; /** @@ -84,17 +74,18 @@ public class LocalTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { + public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { + TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); + return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); + return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); + return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } - return getEmptyTransportApiResponseFuture(); + return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -145,7 +136,7 @@ public class LocalTransportApiService implements TransportApiService { try { ValidateDeviceCredentialsResponseMsg.Builder builder = ValidateDeviceCredentialsResponseMsg.newBuilder(); builder.setDeviceInfo(getDeviceInfoProto(device)); - if(!StringUtils.isEmpty(credentials.getCredentialsValue())){ + if (!StringUtils.isEmpty(credentials.getCredentialsValue())) { builder.setCredentialsBody(credentials.getCredentialsValue()); } return TransportApiResponseMsg.newBuilder() diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 7aa438c64f..83353cea4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -22,9 +22,6 @@ import io.github.bucket4j.Bucket4j; import io.github.bucket4j.local.LocalBucket; import io.github.bucket4j.local.LocalBucketBuilder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -33,15 +30,17 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; @@ -51,6 +50,7 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -84,9 +84,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Value("${transport.remote.rule_engine.stats.enabled:false}") private boolean statsEnabled; - @Autowired - private TbKafkaSettings kafkaSettings; - +// @Autowired +// private TbKafkaSettings kafkaSettings; +// @Autowired private TbNodeIdProvider nodeIdProvider; @@ -101,8 +101,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Autowired private DataDecodingEncodingService encodingService; - private TBKafkaConsumerTemplate ruleEngineConsumer; - private TBKafkaProducerTemplate notificationsProducer; + private TbQueueConsumer> ruleEngineConsumer; + + private TbQueueProducer> notificationsProducer; private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); @@ -146,8 +147,8 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ mainConsumerExecutor.execute(() -> { while (!stopped) { try { - ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); - int recordsCount = records.count(); + List> msgs = ruleEngineConsumer.poll(pollDuration); + int recordsCount = msgs.size(); if (recordsCount > 0) { while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); @@ -155,9 +156,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } log.trace("Processing {} records", recordsCount); } - records.forEach(record -> { + msgs.forEach(msg -> { try { - ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); if (toRuleEngineMsg.hasToDeviceActorMsg()) { forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); @@ -196,7 +197,8 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { @@ -225,7 +227,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } } - private static class QueueCallbackAdaptor implements Callback { + private static class QueueCallbackAdaptor implements TbQueueCallback { private final Runnable onSuccess; private final Consumer onFailure; @@ -235,17 +237,17 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - if (onSuccess != null) { - onSuccess.run(); - } - } else { - if (onFailure != null) { - onFailure.accept(exception); - } + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); } } - } + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 0c9efc7d1c..e354da7daa 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -22,9 +22,16 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.*; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -49,9 +56,9 @@ public class RemoteTransportApiService { @Value("${transport.remote.transport_api.request_auto_commit_interval}") private int autoCommitInterval; - @Autowired - private TbKafkaSettings kafkaSettings; - +// @Autowired +// private TbKafkaSettings kafkaSettings; +// @Autowired private TbNodeIdProvider nodeIdProvider; @@ -60,7 +67,7 @@ public class RemoteTransportApiService { private ExecutorService transportCallbackExecutor; - private TbKafkaResponseTemplate transportApiTemplate; + private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; @PostConstruct public void init() { @@ -79,11 +86,14 @@ public class RemoteTransportApiService { requestBuilder.autoCommit(true); requestBuilder.autoCommitIntervalMs(autoCommitInterval); requestBuilder.decoder(new TransportApiRequestDecoder()); + TbQueueProducer> producer = null; + TbQueueConsumer> consumer = null; + - TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder - builder = TbKafkaResponseTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); + DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); + builder.requestTemplate(consumer); + builder.responseTemplate(producer); builder.maxPendingRequests(maxPendingRequests); builder.requestTimeout(requestTimeout); builder.pollInterval(responsePollDuration); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index 9f08463f91..bdebacbd13 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.transport; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { + @Override - public ToRuleEngineMsg decode(byte[] data) throws IOException { - return ToRuleEngineMsg.parseFrom(data); + public ToRuleEngineMsg decode(TbQueueMsg msg) throws IOException { + return ToRuleEngineMsg.parseFrom(msg.getData()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 2964934313..7d572d08ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,11 +15,12 @@ */ package org.thingsboard.server.service.transport; +import org.thingsboard.server.TbQueueHandler; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.kafka.TbKafkaHandler; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbKafkaHandler { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index 181d50bdd1..ca51eb4ddb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -8,6 +8,8 @@ public interface TbQueueConsumer { void subscribe(); + void unsubscribe(); + List poll(long durationInMillis); void commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index e1a5b4861d..9dad87588d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,8 +1,12 @@ package org.thingsboard.server; +import java.util.Map; + public interface TbQueueMsgHeaders { byte[] put(String key, byte[] value); byte[] get(String key); + + Map getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index 4ef33a489d..e547a6c070 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -7,7 +7,7 @@ import java.util.Map; public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { - private final Map data = new HashMap<>(); + protected final Map data = new HashMap<>(); @Override public byte[] put(String key, byte[] value) { @@ -18,4 +18,9 @@ public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { public byte[] get(String key) { return data.get(key); } + + @Override + public Map getData() { + return data; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index f1120b876f..c072dc4d67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -11,13 +11,13 @@ public class TbProtoQueueMsg i private final UUID key; private final T value; - private final DefaultTbQueueMsgHeaders headers; + private final TbQueueMsgHeaders headers; public TbProtoQueueMsg(UUID key, T value) { this(key, value, new DefaultTbQueueMsgHeaders()); } - public TbProtoQueueMsg(UUID key, T value, DefaultTbQueueMsgHeaders headers) { + public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { this.key = key; this.value = value; this.headers = headers; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java new file mode 100644 index 0000000000..e44a93a70d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -0,0 +1,39 @@ +package org.thingsboard.server.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.common.DefaultTbQueueMsgHeaders; + +import java.util.UUID; + +public class KafkaTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final TbQueueMsgHeaders headers; + private final byte[] data; + + public KafkaTbQueueMsg(ConsumerRecord record) { + this.key = UUID.fromString(record.key()); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + record.headers().forEach(header -> { + headers.put(header.key(), header.value()); + }); + this.headers = headers; + this.data = record.value(); + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return data; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java new file mode 100644 index 0000000000..4698c226d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -0,0 +1,12 @@ +package org.thingsboard.server.kafka; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.thingsboard.server.TbQueueMsgMetadata; + +@Data +@AllArgsConstructor +public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { + private RecordMetadata metadata; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 0fe86e030b..9c6d911d33 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.KafkaFuture; +import org.thingsboard.server.TbQueueAdmin; import java.time.Duration; import java.util.Collections; @@ -30,7 +32,7 @@ import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. */ -public class TBKafkaAdmin { +public class TBKafkaAdmin implements TbQueueAdmin { AdminClient client; @@ -66,4 +68,9 @@ public class TBKafkaAdmin { } } + @Override + public ListenableFuture createTopicIfNotExists(String topic) { + + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 73f8cc16c7..6f0ce7b55c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -17,21 +17,27 @@ package org.thingsboard.server.kafka; import lombok.Builder; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Properties; import java.util.UUID; /** * Created by ashvayka on 24.09.18. */ -public class TBKafkaConsumerTemplate { +@Slf4j +public class TBKafkaConsumerTemplate implements TbQueueConsumer { private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; @@ -66,23 +72,55 @@ public class TBKafkaConsumerTemplate { this.topic = topic; } + @Override public void subscribe() { consumer.subscribe(Collections.singletonList(topic)); } + @Override + public List poll(long durationInMillis) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + + return Collections.emptyList(); + } + + @Override + public void commit() { + consumer.commitAsync(); + } + + @Override public void unsubscribe() { consumer.unsubscribe(); } - public ConsumerRecords poll(Duration duration) { - return consumer.poll(duration); - } +// public void subscribe() { +// consumer.subscribe(Collections.singletonList(topic)); +// } +// + +// +// public ConsumerRecords poll(Duration duration) { +// return consumer.poll(duration); +// } public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(record.value()); + return decoder.decode(new KafkaTbQueueMsg(record)); } public UUID extractRequestId(T value) { return requestIdExtractor.extractRequestId(value); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 8722c1d64e..938b278207 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -15,42 +15,45 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.JdkFutureAdapters; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; import java.util.List; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaProducerTemplate { +public class TBKafkaProducerTemplate implements TbQueueProducer { private final KafkaProducer producer; - private final TbKafkaEncoder encoder; private final TbKafkaPartitioner partitioner; + private ConcurrentMap> partitionInfoMap; + @Getter private final String defaultTopic; @@ -58,8 +61,7 @@ public class TBKafkaProducerTemplate { private final TbKafkaSettings settings; @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, - TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); @@ -68,7 +70,6 @@ public class TBKafkaProducerTemplate { } this.settings = settings; this.producer = new KafkaProducer<>(props); - this.encoder = encoder; this.partitioner = partitioner; this.defaultTopic = defaultTopic; } @@ -89,43 +90,68 @@ public class TBKafkaProducerTemplate { } } - public Future send(String key, T value, Callback callback) { - return send(key, value, null, callback); - } - - public Future send(String key, T value, Iterable
headers, Callback callback) { - return send(key, value, null, headers, callback); + @Override + public ListenableFuture send(T msg, TbQueueCallback callback) { + return send(defaultTopic, msg, callback); } - public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { - if (!StringUtils.isEmpty(this.defaultTopic)) { - return send(this.defaultTopic, key, value, timestamp, headers, callback); - } else { - throw new RuntimeException("Failed to send message! Default topic is not specified!"); - } - } + @Override + public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + String key = msg.getKey().toString(); + byte[] data = msg.getData(); + ProducerRecord record; + Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { - return send(topic, key, value, null, headers, callback); - } + Integer partition = getPartition(topic, msg); + record = new ProducerRecord<>(topic, partition, key, data, headers); + Future result = producer.send(record, (metadata, exception) -> { + if (exception == null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } else { + callback.onFailure(exception); + } + }); - public Future send(String topic, String key, T value, Callback callback) { - return send(topic, key, value, null, null, callback); + return Futures.transform(JdkFutureAdapters.listenInPoolThread(result), metadata -> new KafkaTbQueueMsgMetadata(metadata)); } - public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { - byte[] data = encoder.encode(value); - ProducerRecord record; - Integer partition = getPartition(topic, key, value, data); - record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); - return producer.send(record, callback); - } +// public Future send(String key, T value, Callback callback) { +// return send(key, value, null, callback); +// } +// +// public Future send(String key, T value, Iterable
headers, Callback callback) { +// return send(key, value, null, headers, callback); +// } +// +// public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { +// if (!StringUtils.isEmpty(this.defaultTopic)) { +// return send(this.defaultTopic, key, value, timestamp, headers, callback); +// } else { +// throw new RuntimeException("Failed to send message! Default topic is not specified!"); +// } +// } +// +// public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { +// return send(topic, key, value, null, headers, callback); +// } +// +// public Future send(String topic, String key, T value, Callback callback) { +// return send(topic, key, value, null, null, callback); +// } +// +// public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { +// byte[] data = encoder.encode(value); +// ProducerRecord record; +// Integer partition = getPartition(topic, key, value, data); +// record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); +// return producer.send(record, callback); +// } - private Integer getPartition(String topic, String key, T value, byte[] data) { + private Integer getPartition(String topic, T value) { if (partitioner == null) { return null; } else { - return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); + return partitioner.partition(topic, value.getKey().toString(), value, value.getData(), partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java index ab196d5863..564d905675 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.kafka; +import org.thingsboard.server.TbQueueMsg; + import java.io.IOException; /** @@ -22,6 +24,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(byte[] data) throws IOException; + T decode(TbQueueMsg msg) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java new file mode 100644 index 0000000000..abf30667b5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -0,0 +1,45 @@ +package org.thingsboard.server.memory; + +import org.thingsboard.server.TbQueueMsg; + +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; + +public final class InMemoryStorage { + private static InMemoryStorage instance; + private final Map> storage; + + private InMemoryStorage() { + storage = new ConcurrentHashMap<>(); + } + + public static InMemoryStorage getInstance() { + if (instance == null) { + synchronized (InMemoryStorage.class) { + if (instance == null) { + instance = new InMemoryStorage(); + } + } + } + return instance; + } + + public boolean put(String topic, TbQueueMsg msg) { + return storage.computeIfAbsent(topic, (t) -> new LinkedList<>()).add(msg); + } + + public TbQueueMsg get(String topic) { + if (storage.containsKey(topic)) { + return storage.get(topic).peek(); + } + return null; + } + + public void commit(String topic) { + if (storage.containsKey(topic)) { + storage.get(topic).remove(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java new file mode 100644 index 0000000000..a0a35eefb8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.memory; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; + +import java.util.Collections; +import java.util.List; + +public class InMemoryTbQueueConsumer implements TbQueueConsumer { + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + public InMemoryTbQueueConsumer(String topic) { + this.topic = topic; + } + + private final String topic; + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + + } + + @Override + public void unsubscribe() { + + } + + @Override + public List poll(long durationInMillis) { + return Collections.singletonList((T)storage.get(topic)); + } + + @Override + public void commit() { + storage.commit(topic); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java new file mode 100644 index 0000000000..3b53ddf316 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -0,0 +1,43 @@ +package org.thingsboard.server.memory; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; + +public class InMemoryTbQueueProducer implements TbQueueProducer { + + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + private String defaultTopic; + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public ListenableFuture send(T msg, TbQueueCallback callback) { + return send(defaultTopic, msg, callback); + } + + @Override + public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + boolean result = storage.put(topic, msg); + if (result) { + callback.onSuccess(null); + return Futures.immediateCheckedFuture(null); + } else { + Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); + callback.onFailure(e); + return Futures.immediateFailedFuture(e); + } + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index dd7d9bd27a..921003dbd7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -40,7 +40,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import org.thingsboard.server.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index c07e406d2e..31a32d8368 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { + @Override - public ToTransportMsg decode(byte[] data) throws IOException { - return ToTransportMsg.parseFrom(data); + public ToTransportMsg decode(TbQueueMsg msg) throws IOException { + return ToTransportMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index a62e696500..89d61fba56 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { + @Override - public TransportApiResponseMsg decode(byte[] data) throws IOException { - return TransportApiResponseMsg.parseFrom(data); + public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { + return TransportApiResponseMsg.parseFrom(msg.getData()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index c49fcc3728..5867e505b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -18,12 +18,21 @@ package org.thingsboard.server.dao.entity; import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.alarm.AlarmId; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; @@ -109,7 +118,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe default: throw new IllegalStateException("Not Implemented!"); } - entityName = Futures.transform(hasName, (Function) hasName1 -> hasName1 != null ? hasName1.getName() : null ); + entityName = Futures.transform(hasName, (Function) hasName1 -> hasName1 != null ? hasName1.getName() : null, MoreExecutors.directExecutor()); return entityName; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 68b7fce6a8..22b2543489 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,7 +16,10 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.Cache; @@ -206,17 +209,20 @@ public class BaseRelationService implements RelationService { relations -> { List> results = deleteRelationGroupsAsync(tenantId, relations, cache, true); return Futures.allAsList(results); - }); + }, MoreExecutors.directExecutor()); ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, relations -> { List> results = deleteRelationGroupsAsync(tenantId, relations, cache, false); return Futures.allAsList(results); - }); + }, MoreExecutors.directExecutor()); ListenableFuture>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions); - return Futures.transform(Futures.transformAsync(deletionsFuture, (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId)), result -> null); + return Futures.transform(Futures.transformAsync(deletionsFuture, + (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId), + MoreExecutors.directExecutor()), + result -> null, MoreExecutors.directExecutor()); } private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, Cache cache, boolean deleteFromDb) { @@ -306,9 +312,11 @@ public class BaseRelationService implements RelationService { public void onSuccess(@Nullable List result) { cache.putIfAbsent(fromAndTypeGroup, result); } + @Override - public void onFailure(Throwable t) {} - }); + public void onFailure(Throwable t) { + } + }, MoreExecutors.directExecutor()); return relationsFuture; } } @@ -328,7 +336,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo::setToName)) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #relationType, #typeGroup, 'FROM'}") @@ -385,9 +393,11 @@ public class BaseRelationService implements RelationService { public void onSuccess(@Nullable List result) { cache.putIfAbsent(toAndTypeGroup, result); } + @Override - public void onFailure(Throwable t) {} - }); + public void onFailure(Throwable t) { + } + }, MoreExecutors.directExecutor()); return relationsFuture; } } @@ -407,7 +417,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo::setFromName)) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } private ListenableFuture fetchRelationInfoAsync(TenantId tenantId, EntityRelation relation, @@ -418,7 +428,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo entityRelationInfo1 = new EntityRelationInfo(relation); entityNameSetter.accept(entityRelationInfo1, entityName1); return entityRelationInfo1; - }); + }, MoreExecutors.directExecutor()); } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#to, #relationType, #typeGroup, 'TO'}") @@ -466,7 +476,7 @@ public class BaseRelationService implements RelationService { } } return relations; - }); + }, MoreExecutors.directExecutor()); } catch (Exception e) { log.warn("Failed to query relations: [{}]", query, e); throw new RuntimeException(e); @@ -493,7 +503,7 @@ public class BaseRelationService implements RelationService { })) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } protected void validate(EntityRelation relation) { @@ -600,7 +610,7 @@ public class BaseRelationService implements RelationService { } //TODO: try to remove this blocking operation List> relations = Futures.successfulAsList(futures).get(); - if (fetchLastLevelOnly && lvl > 0){ + if (fetchLastLevelOnly && lvl > 0) { children.clear(); } relations.forEach(r -> r.forEach(children::add)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 176bc712e8..4ca53a337b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sqlts.timescale; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -143,7 +144,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } else { return Collections.emptyList(); } - }); + }, MoreExecutors.directExecutor()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 8fc8b4ab8a..b96462e350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -330,7 +331,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem stmt.setInt(6, (int) ttl); } futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null); + return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); } private void processSetNullValues(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, List> futures, long partition, DataType type) { @@ -545,7 +546,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem public void onFailure(Throwable t) { log.warn("[{}] Failed to process remove of the latest value", entityId, t); } - }); + }, MoreExecutors.directExecutor()); return resultFuture; } From 6e31c0ab157172897bcacd6e1b22b049302400d8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 11 Mar 2020 20:09:41 +0200 Subject: [PATCH 006/102] Refactoring of configuration properties --- .../src/main/resources/thingsboard.yml | 75 +++++++++---------- .../memory/InMemoryTbQueueProducer.java | 4 +- .../queue/KafkaTransportQueueProvider.java | 4 + .../service/DefaultTransportService.java | 38 ++++++---- .../src/main/resources/tb-mqtt-transport.yml | 34 +++++---- 5 files changed, 83 insertions(+), 72 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 820fe14be1..0cd5be8fc8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -428,28 +428,6 @@ state: defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -489,25 +467,6 @@ js: transport: type: "${TRANSPORT_TYPE:local}" # local or remote - remote: - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" - max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" - max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" - stats: - enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -573,3 +532,37 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" + + +queue: + type: "${TB_QUEUE_TYPE:kafka}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + rule_engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + notifications: + topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + +rule_engine: + type: "${RULE_ENGINE_TYPE:local}" # local or remote \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 3b53ddf316..32b9399164 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -2,16 +2,18 @@ package org.thingsboard.server.memory; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; +@Data public class InMemoryTbQueueProducer implements TbQueueProducer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); - private String defaultTopic; + private final String defaultTopic; @Override public void init() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java index 2aa6f90b7b..e16c7fd93c 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java @@ -1,5 +1,7 @@ package org.thingsboard.server.common.transport.queue; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; @@ -8,6 +10,8 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; @Component +@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 921003dbd7..9ccb5369f8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -192,7 +192,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscriptionInfo(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } @Override @@ -203,7 +203,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSessionEvent(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -215,7 +215,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setPostTelemetry(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @@ -227,7 +227,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setPostAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @@ -239,7 +239,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setGetAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -252,7 +252,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscribeToAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -265,7 +265,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscribeToRPC(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -277,10 +277,11 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setToDeviceRPCCallResponse(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } + //TODO 2.5: Need to handle timeouts on the transport level and not on the Device Actor Level. @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { @@ -289,18 +290,20 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setToServerRPCCallRequest(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + if (checkLimits(sessionInfo, msg, callback)) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + } } @Override @@ -451,7 +454,12 @@ public class DefaultTransportService implements TransportService { .setEvent(event).build(); } - protected void send(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? new TransportTbQueueCallback(callback) : null); } diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 47dfe57c85..741adc6795 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -61,24 +61,28 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - queue: - type: kafka + +queue: + type: "${TB_QUEUE_TYPE:kafka}" + kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + rule_engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + notifications: + topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" From 52814d2bfcb7e11feb44ccf26a81fc96bd337637 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 09:36:48 +0200 Subject: [PATCH 007/102] Refactoring of Transport Communication to Queues --- ...e.java => DefaultTransportApiService.java} | 2 +- .../transport/RemoteTransportApiService.java | 57 ++++------- .../transport/TransportApiService.java | 5 +- .../src/main/resources/thingsboard.yml | 1 + common/queue/pom.xml | 9 ++ .../provider/KafkaTbCoreQueueProvider.java | 42 ++++++++ .../provider/KafkaTransportQueueProvider.java | 39 ++++++++ .../server/provider/TbCoreQueueProvider.java | 63 ++++++++++++ .../provider}/TransportQueueProvider.java | 5 +- .../src/main/proto/transport.proto | 72 +++++++++----- common/transport/transport-api/pom.xml | 13 --- .../queue/KafkaTransportQueueProvider.java | 35 ------- .../service/DefaultTransportService.java | 98 +++++++------------ 13 files changed, 260 insertions(+), 181 deletions(-) rename application/src/main/java/org/thingsboard/server/service/transport/{LocalTransportApiService.java => DefaultTransportApiService.java} (99%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue => queue/src/main/java/org/thingsboard/server/provider}/TransportQueueProvider.java (82%) rename common/{transport/transport-api => queue}/src/main/proto/transport.proto (75%) delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index f6434036fd..51597877b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -52,7 +52,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -public class LocalTransportApiService implements TransportApiService { +public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index e354da7daa..644d4191b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,6 +32,7 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.provider.TbCoreQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -45,50 +46,32 @@ import java.util.concurrent.*; @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { - @Value("${transport.remote.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${transport.remote.transport_api.max_pending_requests}") + private final TbCoreQueueProvider tbCoreQueueProvider; + private final TransportApiService transportApiService; + + @Value("${queue.transport_api.max_pending_requests:10000}") private int maxPendingRequests; - @Value("${transport.remote.transport_api.request_timeout}") + @Value("${queue.transport_api.max_requests_timeout:10000}") private long requestTimeout; - @Value("${transport.remote.transport_api.request_poll_interval}") + @Value("${queue.transport_api.request_poll_interval:25}") private int responsePollDuration; - @Value("${transport.remote.transport_api.request_auto_commit_interval}") - private int autoCommitInterval; - -// @Autowired -// private TbKafkaSettings kafkaSettings; -// - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TransportApiService transportApiService; + @Value("${queue.transport_api.max_callback_threads:100}") + private int maxCallbackThreads; private ExecutorService transportCallbackExecutor; + private TbQueueResponseTemplate, + TbProtoQueueMsg> transportApiTemplate; - private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; + public RemoteTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { + this.tbCoreQueueProvider = tbCoreQueueProvider; + this.transportApiService = transportApiService; + } @PostConstruct public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(100); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); - responseBuilder.encoder(new TransportApiResponseEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.topic(transportApiRequestsTopic); - requestBuilder.clientId(nodeIdProvider.getNodeId()); - requestBuilder.groupId("tb-node"); - requestBuilder.autoCommit(true); - requestBuilder.autoCommitIntervalMs(autoCommitInterval); - requestBuilder.decoder(new TransportApiRequestDecoder()); - TbQueueProducer> producer = null; - TbQueueConsumer> consumer = null; - + this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); + TbQueueProducer> producer = tbCoreQueueProvider.getTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueProvider.getTransportApiRequestConsumer(); DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); @@ -105,7 +88,7 @@ public class RemoteTransportApiService { @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(); + transportApiTemplate.init(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 7d572d08ed..eab2e07b0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -17,10 +17,11 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.TbQueueHandler; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cd5be8fc8..dc479191e7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -548,6 +548,7 @@ queue: responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 952b3c540d..fd26358cc6 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -100,4 +100,13 @@ + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java new file mode 100644 index 0000000000..eac81b80c3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +@Component +@ConditionalOnExpression("'${transport.type:null}'=='local' || '${transport.type:null}'=='remote'") +public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ + @Override + public TbQueueProducer> getTransportMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return null; + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java new file mode 100644 index 0000000000..6686b3704f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -0,0 +1,39 @@ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +@Component +@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@Slf4j +public class KafkaTransportQueueProvider implements TransportQueueProvider { + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java new file mode 100644 index 0000000000..b7e210f0f6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -0,0 +1,63 @@ +package org.thingsboard.server.provider; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbCoreQueueProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToCoreMsgConsumer(); + + /** + * Used to consume Transport API Calls + * + * @return + */ + TbQueueConsumer> getTransportApiRequestConsumer(); + + /** + * Used to push replies to Transport API Calls + * + * @return + */ + TbQueueProducer> getTransportApiResponseProducer(); + + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java similarity index 82% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index d0bbafc615..14d8efd534 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,9 +1,10 @@ -package org.thingsboard.server.common.transport.queue; +package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; @@ -15,7 +16,7 @@ public interface TransportQueueProvider { TbQueueProducer> getRuleEngineMsgProducer(); - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> getTbCoreMsgProducer(); TbQueueConsumer> getTransportNotificationsConsumer(); diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto similarity index 75% rename from common/transport/transport-api/src/main/proto/transport.proto rename to common/queue/src/main/proto/transport.proto index e8b513574a..807aa571bf 100644 --- a/common/transport/transport-api/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -20,16 +20,18 @@ option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; /** - * Data Structures; + * Transport Service Data Structures; */ message SessionInfoProto { - string nodeId = 1; + string ServiceId = 1; int64 sessionIdMSB = 2; int64 sessionIdLSB = 3; int64 tenantIdMSB = 4; int64 tenantIdLSB = 5; int64 deviceIdMSB = 6; int64 deviceIdLSB = 7; + string deviceName = 8; + string deviceType = 9; } enum SessionEvent { @@ -81,7 +83,7 @@ message DeviceInfoProto { } /** - * Messages that use Data Structures; + * Transport Service Messages; */ message SessionEventMsg { SessionType sessionType = 1; @@ -181,7 +183,7 @@ message ClaimDeviceMsg { int64 durationMs = 4; } -//Used to report session state to tb-node and persist this state in the cache on the tb-node level. +//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; bool attributeSubscription = 2; @@ -200,45 +202,61 @@ message DeviceSessionsCacheEntry { message TransportToDeviceActorMsg { SessionInfoProto sessionInfo = 1; SessionEventMsg sessionEvent = 2; - PostTelemetryMsg postTelemetry = 3; - PostAttributeMsg postAttributes = 4; - GetAttributeRequestMsg getAttributes = 5; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; - SubscribeToRPCMsg subscribeToRPC = 7; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; - ToServerRpcRequestMsg toServerRPCCallRequest = 9; - SubscriptionInfoProto subscriptionInfo = 10; - ClaimDeviceMsg claimDevice = 11; + GetAttributeRequestMsg getAttributes = 3; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; + SubscribeToRPCMsg subscribeToRPC = 5; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; + SubscriptionInfoProto subscriptionInfo = 7; + ClaimDeviceMsg claimDevice = 8; +} + +message TransportToRuleEngineMsg { + SessionInfoProto sessionInfo = 1; + PostTelemetryMsg postTelemetry = 2; + PostAttributeMsg postAttributes = 3; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; + ToServerRpcRequestMsg toServerRPCCallRequest = 5; + SubscriptionInfoProto subscriptionInfo = 6; } message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } /** * Main messages; */ -message ToRuleEngineMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; -} - -message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; -} +/* Request from Transport Service to ThingsBoard Core Service */ message TransportApiRequestMsg { ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; } +/* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; } + +/* Messages that are handled by ThingsBoard Core Service */ +message ToCoreMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; +} + +/* Messages that are handled by ThingsBoard RuleEngine Service */ +message ToRuleEngineMsg { + TransportToRuleEngineMsg toRuleEngineMsg = 1; +} + +/* Messages that are handled by ThingsBoard Transport Service */ +message ToTransportMsg { + DeviceActorToTransportMsg toDeviceSessionMsg = 1; +} diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 3920234f7c..d6dcbbabc6 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -99,19 +99,6 @@ org.apache.commons commons-lang3 - - com.google.protobuf - protobuf-java - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java deleted file mode 100644 index e16c7fd93c..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; - -@Component -@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") -@Slf4j -public class KafkaTransportQueueProvider implements TransportQueueProvider { - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { - return null; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return null; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return null; - } - - @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { - return null; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9ccb5369f8..e4dc2ba1f1 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -34,12 +34,15 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.queue.TransportQueueProvider; +import org.thingsboard.server.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; import org.thingsboard.server.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; @@ -81,7 +84,7 @@ public class DefaultTransportService implements TransportService { protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; + protected TbQueueProducer> tbCoreMsgProducer; protected TbQueueConsumer> transportNotificationsConsumer; protected ScheduledExecutorService schedulerExecutor; @@ -188,22 +191,16 @@ public class DefaultTransportService implements TransportService { if (log.isTraceEnabled()) { log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build(), callback); } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build(), callback); } } @@ -211,11 +208,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setPostTelemetry(msg).build(), callback); } } @@ -223,11 +217,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setPostAttributes(msg).build(), callback); } } @@ -235,11 +226,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build(), callback); } } @@ -248,11 +236,8 @@ public class DefaultTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build(), callback); } } @@ -261,11 +246,8 @@ public class DefaultTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build(), callback); } } @@ -273,11 +255,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build(), callback); } } @@ -286,23 +265,16 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setToServerRPCCallRequest(msg).build(), callback); } } @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, - TransportServiceCallback callback) { + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build(), callback); } } @@ -454,13 +426,15 @@ public class DefaultTransportService implements TransportService { .setEvent(event).build(); } - protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? new TransportTbQueueCallback(callback) : null); } - protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? new TransportTbQueueCallback(callback) : null); } @@ -473,16 +447,12 @@ public class DefaultTransportService implements TransportService { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); } @Override public void onFailure(Throwable t) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onError(t); - }); + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); } } } From fa9194c1c13f3382cd7404143db2fdbb9d28aae8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 11:11:56 +0200 Subject: [PATCH 008/102] Core consumer service implementation --- .../queue/DefaultTbCoreConsumerService.java | 126 ++++++++++++++++++ .../server/service/queue/MsgPackCallback.java | 37 +++++ .../service/queue/TbCoreConsumerService.java | 9 ++ .../server/service/queue/TbMsgCallback.java | 9 ++ .../DefaultTbCoreToTransportService.java | 72 ++++++++++ .../transport/RemoteTransportApiService.java | 3 +- .../transport/RuleEngineTransportService.java | 31 ----- .../transport/TbCoreToTransportService.java | 13 ++ .../msg/TransportToDeviceActorMsgWrapper.java | 5 +- .../src/main/resources/thingsboard.yml | 10 +- .../provider/KafkaTbCoreQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 3 + 13 files changed, 283 insertions(+), 39 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java new file mode 100644 index 0000000000..d919857318 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -0,0 +1,126 @@ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.transport.RuleEngineStats; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@Slf4j +public class DefaultTbCoreConsumerService implements TbCoreConsumerService { + + @Value("${queue.core.poll_interval}") + private long pollDuration; + @Value("${queue.core.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.core.stats.enabled:false}") + private boolean statsEnabled; + + private final ActorSystemContext actorContext; + private final TbQueueConsumer> consumer; + private final RuleEngineStats stats = new RuleEngineStats(); + private volatile ExecutorService mainConsumerExecutor; + private volatile boolean stopped = false; + + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.actorContext = actorContext; + } + + @PostConstruct + public void init() { + this.consumer.subscribe(); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + ackMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + try { + ToCoreMsg toRuleEngineMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java new file mode 100644 index 0000000000..c700c18dc0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class MsgPackCallback implements TbMsgCallback { + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap> ackMap; + private final UUID id; + + public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, ConcurrentMap> ackMap) { + this.id = id; + this.processingTimeoutLatch = processingTimeoutLatch; + this.ackMap = ackMap; + } + + @Override + public void onSuccess() { + if (ackMap.remove(id) != null && ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } + + @Override + public void onFailure(Throwable t) { + TbProtoQueueMsg message = ackMap.remove(id); + log.warn("Failed to process message: {}", message.getValue(), t); + if (ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java new file mode 100644 index 0000000000..0541de7c63 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue; + +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.function.Consumer; + +public interface TbCoreConsumerService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java new file mode 100644 index 0000000000..44dd058e1e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue; + +public interface TbMsgCallback { + + void onSuccess(); + + void onFailure(Throwable t); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java new file mode 100644 index 0000000000..c1b0608d44 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -0,0 +1,72 @@ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.provider.TbCoreQueueProvider; + +import java.util.UUID; +import java.util.function.Consumer; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + + private final TbCoreQueueProvider tbCoreQueueProvider; + private final TbQueueProducer> tbTransportProducer; + + @Value("${queue.notifications.topic}") + private String notificationsTopic; + + public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { + this.tbCoreQueueProvider = tbCoreQueueProvider; + this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + String topic = notificationsTopic + "." + nodeId; + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); + log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private static class QueueCallbackAdaptor implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 644d4191b1..354252642f 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -22,6 +22,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; @@ -42,7 +43,7 @@ import java.util.concurrent.*; * Created by ashvayka on 05.10.18. */ @Slf4j -@Component +@Service @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java deleted file mode 100644 index 0d401e4d9c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 05.10.18. - */ -public interface RuleEngineTransportService { - - void process(String nodeId, DeviceActorToTransportMsg msg); - - void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java new file mode 100644 index 0000000000..3324f02440 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -0,0 +1,13 @@ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.function.Consumer; + +public interface TbCoreToTransportService { + + void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg); + + void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 34abac72bb..880d7507ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.service.queue.TbMsgCallback; import java.io.Serializable; import java.util.UUID; @@ -36,9 +37,11 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; + private final TbMsgCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbMsgCallback callback) { this.msg = msg; + this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index dc479191e7..2ded608203 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -466,7 +466,6 @@ js: print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: - type: "${TRANSPORT_TYPE:local}" # local or remote sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -533,7 +532,6 @@ swagger: url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" - queue: type: "${TB_QUEUE_TYPE:kafka}" kafka: @@ -555,6 +553,10 @@ queue: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -565,5 +567,5 @@ queue: notifications: topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" -rule_engine: - type: "${RULE_ENGINE_TYPE:local}" # local or remote \ No newline at end of file +service: + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index eac81b80c3..2f9a9ad8f8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -8,7 +8,7 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; @Component -@ConditionalOnExpression("'${transport.type:null}'=='local' || '${transport.type:null}'=='remote'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && ${queue.type:null}'=='kafka'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ @Override public TbQueueProducer> getTransportMsgProducer() { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 6686b3704f..945888a45a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -14,7 +14,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; @Component -@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && ${queue.type:null}'=='kafka'") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @Override diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 741adc6795..721db2605d 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -62,6 +62,9 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" +service: + type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + queue: type: "${TB_QUEUE_TYPE:kafka}" kafka: From c4c53bfbd8ed09c9e533318f73fb55654ff5381a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 14:37:38 +0200 Subject: [PATCH 009/102] Transport Implementation --- .../server/actors/ActorSystemContext.java | 9 +- .../device/DeviceActorMessageProcessor.java | 35 ++-- .../actors/service/DefaultActorService.java | 3 - .../cluster/rpc/ClusterGrpcService.java | 161 ------------------ .../cluster/rpc/ClusterRpcService.java | 42 ----- .../service/cluster/rpc/GrpcSession.java | 125 -------------- .../cluster/rpc/GrpcSessionListener.java | 32 ---- .../service/cluster/rpc/RpcMsgListener.java | 32 ---- .../queue/DefaultTbCoreConsumerService.java | 11 +- .../DefaultTbRuleEngineConsumerService.java | 124 ++++++++++++++ .../service/queue/TbCoreConsumerStats.java | 70 ++++++++ .../queue/TbRuleEngineConsumerService.java | 5 + .../TbRuleEngineConsumerStats.java} | 4 +- .../RemoteRuleEngineTransportService.java | 3 +- common/queue/src/main/proto/transport.proto | 2 +- 15 files changed, 229 insertions(+), 429 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java rename application/src/main/java/org/thingsboard/server/service/{transport/RuleEngineStats.java => queue/TbRuleEngineConsumerStats.java} (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index bf06a3a8ac..d10436075b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -67,7 +67,6 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -81,7 +80,7 @@ import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.RuleEngineTransportService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -122,10 +121,6 @@ public class ActorSystemContext { @Getter private ClusterRoutingService routingService; - @Autowired - @Getter - private ClusterRpcService rpcService; - @Autowired @Getter private DataDecodingEncodingService encodingService; @@ -241,7 +236,7 @@ public class ActorSystemContext { @Lazy @Autowired @Getter - private RuleEngineTransportService ruleEngineTransportService; + private TbCoreToTransportService tbCoreToTransportService; @Lazy @Autowired diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 2752b53090..6c0dde4044 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -234,24 +234,24 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } - if (msg.hasPostAttributes()) { - handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); - reportDeviceActivity = true; - } - if (msg.hasPostTelemetry()) { - handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); - reportDeviceActivity = true; - } +// if (msg.hasPostAttributes()) { +// handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); +// reportDeviceActivity = true; +// } +// if (msg.hasPostTelemetry()) { +// handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); +// reportDeviceActivity = true; +// } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } - if (msg.hasToServerRPCCallRequest()) { - handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); - reportDeviceActivity = true; - } +// if (msg.hasToServerRPCCallRequest()) { +// handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); +// reportDeviceActivity = true; +// } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } @@ -260,6 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } + //TODO: 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } @@ -540,7 +541,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -556,7 +557,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { @@ -564,7 +565,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { @@ -572,7 +573,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { @@ -580,7 +581,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 43344ef374..e5da93b9e2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -52,15 +52,12 @@ import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.cluster.discovery.ServerInstance; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.transport.RuleEngineStats; import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.duration.Duration; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java deleted file mode 100644 index 23b050634c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { - - @Autowired - private ServerInstanceService instanceService; - - @Autowired - private DataDecodingEncodingService encodingService; - - private RpcMsgListener listener; - - private Server server; - - private ServerInstance instance; - - private ConcurrentMap>> pendingSessionMap = - new ConcurrentHashMap<>(); - - public void init(RpcMsgListener listener) { - this.listener = listener; - log.info("Initializing RPC service!"); - instance = instanceService.getSelf(); - server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); - log.info("Going to start RPC server using port: {}", instance.getPort()); - try { - server.start(); - } catch (IOException e) { - log.error("Failed to start RPC server!", e); - throw new RuntimeException("Failed to start RPC server!"); - } - log.info("RPC service initialized!"); - } - - @Override - public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { - BlockingQueue> queue = pendingSessionMap.remove(msgUid); - if (queue != null) { - try { - queue.put(inputStream); - } catch (InterruptedException e) { - log.warn("Failed to report created session!"); - Thread.currentThread().interrupt(); - } - } else { - log.warn("Failed to lookup pending session!"); - } - } - - @Override - public StreamObserver handleMsgs( - StreamObserver responseObserver) { - log.info("Processing new session."); - return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); - } - - - @PreDestroy - public void stop() { - if (server != null) { - log.info("Going to onStop RPC server"); - server.shutdownNow(); - try { - server.awaitTermination(); - log.info("RPC server stopped!"); - } catch (InterruptedException e) { - log.warn("Failed to onStop RPC server!"); - Thread.currentThread().interrupt(); - } - } - } - - - @Override - public void broadcast(RpcBroadcastMsg msg) { - listener.onBroadcastMsg(msg); - } - - private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { - BlockingQueue> queue = new ArrayBlockingQueue<>(1); - pendingSessionMap.put(msg.getMsgUid(), queue); - listener.onRpcSessionCreateRequestMsg(msg); - try { - StreamObserver observer = queue.take(); - log.info("Processed new session."); - return observer; - } catch (Exception e) { - log.info("Failed to process session.", e); - throw new RuntimeException(e); - } - } - - @Override - public void tell(ClusterAPIProtos.ClusterMessage message) { - listener.onSendMsg(message); - } - - @Override - public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { - listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); - } - - @Override - public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { - ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(msgType) - .setPayload(ByteString.copyFrom(data)).build(); - listener.onSendMsg(msg); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java deleted file mode 100644 index 637924fca9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.stub.StreamObserver; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRpcService { - - void init(RpcMsgListener listener); - - void broadcast(RpcBroadcastMsg msg); - - void onSessionCreated(UUID msgUid, StreamObserver inputStream); - - void tell(ClusterAPIProtos.ClusterMessage message); - - void tell(ServerAddress serverAddress, TbActorMsg actorMsg); - - void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java deleted file mode 100644 index aff0bc4a41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.io.Closeable; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -@Slf4j -public final class GrpcSession implements Closeable { - private final UUID sessionId; - private final boolean client; - private final GrpcSessionListener listener; - private final ManagedChannel channel; - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private ServerAddress remoteServer; - - public GrpcSession(GrpcSessionListener listener) { - this(null, listener, null); - } - - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { - this.sessionId = UUID.randomUUID(); - this.listener = listener; - if (remoteServer != null) { - this.client = true; - this.connected = true; - this.remoteServer = remoteServer; - } else { - this.client = false; - } - this.channel = channel; - } - - public void initInputStream() { - this.inputStream = new StreamObserver() { - @Override - public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { - if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { - connected = true; - ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); - remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); - listener.onConnected(GrpcSession.this); - } - if (connected) { - listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); - } - } - - @Override - public void onError(Throwable t) { - listener.onError(GrpcSession.this, t); - } - - @Override - public void onCompleted() { - outputStream.onCompleted(); - listener.onDisconnected(GrpcSession.this); - } - }; - } - - public void initOutputStream() { - if (client) { - listener.onConnected(GrpcSession.this); - } - } - - public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (connected) { - try { - outputStream.onNext(msg); - } catch (Throwable t) { - try { - outputStream.onError(t); - } catch (Throwable t2) { - } - listener.onError(GrpcSession.this, t); - } - } else { - log.warn("[{}] Failed to send message due to closed session!", sessionId); - } - } - - @Override - public void close() { - connected = false; - try { - outputStream.onCompleted(); - } catch (IllegalStateException e) { - log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); - } - if (channel != null) { - channel.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java deleted file mode 100644 index a6ecf967d6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -public interface GrpcSessionListener { - - void onConnected(GrpcSession session); - - void onDisconnected(GrpcSession session); - - void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); - - void onError(GrpcSession session, Throwable t); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java deleted file mode 100644 index 2ff37b39e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ - -public interface RpcMsgListener { - void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); - void onSendMsg(ClusterAPIProtos.ClusterMessage msg); - void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); - void onBroadcastMsg(RpcBroadcastMsg msg); -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d919857318..4cd5c664fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -15,7 +15,6 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.transport.RuleEngineStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -44,7 +43,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final TbQueueConsumer> consumer; - private final RuleEngineStats stats = new RuleEngineStats(); + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; @@ -71,10 +70,10 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { ackMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { - ToCoreMsg toRuleEngineMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg(), callback); + ToCoreMsg toCoreMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toCoreMsg); + if (toCoreMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); } else { callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java new file mode 100644 index 0000000000..96b72843ed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -0,0 +1,124 @@ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +@Slf4j +public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { + + + @Value("${queue.rule-engine.poll_interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:false}") + private boolean statsEnabled; + + private final ActorSystemContext actorContext; + private final TbQueueConsumer> consumer; + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + private volatile ExecutorService mainConsumerExecutor; + private volatile boolean stopped = false; + + public DefaultTbRuleEngineConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.actorContext = actorContext; + } + + @PostConstruct + public void init() { + this.consumer.subscribe(); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + ackMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + try { + TransportProtos.ToCoreMsg toCoreMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toCoreMsg); + if (toCoreMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void forwardToDeviceActor(TransportProtos.TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java new file mode 100644 index 0000000000..e3bb3a3c4b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbCoreConsumerStats { + + private final AtomicInteger totalCounter = new AtomicInteger(0); + private final AtomicInteger sessionEventCounter = new AtomicInteger(0); + private final AtomicInteger getAttributesCounter = new AtomicInteger(0); + private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); + private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); + private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); + private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); + private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + + public void log(TransportProtos.TransportToDeviceActorMsg msg) { + totalCounter.incrementAndGet(); + if (msg.hasSessionEvent()) { + sessionEventCounter.incrementAndGet(); + } + if (msg.hasGetAttributes()) { + getAttributesCounter.incrementAndGet(); + } + if (msg.hasSubscribeToAttributes()) { + subscribeToAttributesCounter.incrementAndGet(); + } + if (msg.hasSubscribeToRPC()) { + subscribeToRPCCounter.incrementAndGet(); + } + if (msg.hasToDeviceRPCCallResponse()) { + toDeviceRPCCallResponseCounter.incrementAndGet(); + } + if (msg.hasSubscriptionInfo()) { + subscriptionInfoCounter.incrementAndGet(); + } + if (msg.hasClaimDevice()) { + claimDeviceCounter.incrementAndGet(); + } + } + + public void printStats() { + int total = totalCounter.getAndSet(0); + if (total > 0) { + log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]", + total, sessionEventCounter.getAndSet(0), + getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java new file mode 100644 index 0000000000..691a54aaab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -0,0 +1,5 @@ +package org.thingsboard.server.service.queue; + +public interface TbRuleEngineConsumerService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index dea6e6d33c..80d05628f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,7 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class RuleEngineStats { +public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 83353cea4b..980f5e6964 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.TbCoreConsumerStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -109,7 +110,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ private volatile boolean stopped = false; - private final RuleEngineStats stats = new RuleEngineStats(); + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); @PostConstruct public void init() { diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto index 807aa571bf..892139a84a 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -23,7 +23,7 @@ option java_outer_classname = "TransportProtos"; * Transport Service Data Structures; */ message SessionInfoProto { - string ServiceId = 1; + string nodeId = 1; int64 sessionIdMSB = 2; int64 sessionIdLSB = 3; int64 tenantIdMSB = 4; From dc6081d820b236a2ad963b8a7c2bcd1c54bfd89a Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Fri, 13 Mar 2020 17:31:58 +0200 Subject: [PATCH 010/102] Created kafka and in-memory queue providers * Created kafka and in-memory queue providers * Added header requestTime * refactored --- .../server/actors/service/ActorService.java | 1 - .../queue/DefaultTbCoreConsumerService.java | 15 ++ .../DefaultTbRuleEngineConsumerService.java | 15 ++ .../server/service/queue/MsgPackCallback.java | 15 ++ .../service/queue/TbCoreConsumerService.java | 15 ++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../server/service/queue/TbMsgCallback.java | 15 ++ .../queue/TbRuleEngineConsumerService.java | 15 ++ .../queue/TbRuleEngineConsumerStats.java | 24 +- .../service/script/RemoteJsInvokeService.java | 8 - .../DefaultTbCoreToTransportService.java | 19 +- .../RemoteRuleEngineTransportService.java | 37 +-- .../transport/RemoteTransportApiService.java | 2 +- .../transport/TbCoreToTransportService.java | 15 ++ .../org/thingsboard/server/TbQueueAdmin.java | 15 ++ .../thingsboard/server/TbQueueCallback.java | 15 ++ .../thingsboard/server/TbQueueConsumer.java | 15 ++ ...xtractor.java => TbQueueCoreSettings.java} | 15 +- .../org/thingsboard/server/TbQueueMsg.java | 15 ++ .../thingsboard/server/TbQueueMsgHeaders.java | 15 ++ .../server/TbQueueMsgMetadata.java | 15 ++ .../thingsboard/server/TbQueueProducer.java | 21 +- .../server/TbQueueRequestTemplate.java | 15 ++ .../server/TbQueueResponseTemplate.java | 15 ++ .../server/TbQueueTransportApiSettings.java | 40 ++++ .../common/AbstractTbQueueTemplate.java | 29 +++ .../common/DefaultTbQueueMsgHeaders.java | 15 ++ .../common/DefaultTbQueueRequestTemplate.java | 18 +- .../DefaultTbQueueResponseTemplate.java | 88 +++++--- .../server/common/TbProtoQueueMsg.java | 15 ++ .../server/kafka/AbstractTbKafkaTemplate.java | 50 ---- .../server/kafka/AsyncCallbackTemplate.java | 66 ------ .../server/kafka/KafkaTbQueueMsg.java | 15 ++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 15 ++ .../server/kafka/TBKafkaAdmin.java | 32 ++- .../server/kafka/TBKafkaConsumerTemplate.java | 20 -- .../server/kafka/TBKafkaProducerTemplate.java | 44 +--- .../server/kafka/TbKafkaRequestTemplate.java | 213 ------------------ .../server/kafka/TbKafkaResponseTemplate.java | 161 ------------- .../server/kafka/TbKafkaSettings.java | 12 +- .../server/memory/InMemoryStorage.java | 15 ++ .../memory/InMemoryTbQueueConsumer.java | 15 ++ .../memory/InMemoryTbQueueProducer.java | 30 ++- .../provider/InMemoryTbCoreQueueProvider.java | 79 +++++++ .../InMemoryTransportQueueProvider.java | 79 +++++++ .../provider/KafkaTbCoreQueueProvider.java | 88 ++++++-- .../provider/KafkaTransportQueueProvider.java | 70 +++++- .../server/provider/TbCoreQueueProvider.java | 17 +- .../provider/TransportQueueProvider.java | 15 ++ .../service/DefaultTransportService.java | 2 +- 50 files changed, 905 insertions(+), 692 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaRequestIdExtractor.java => TbQueueCoreSettings.java} (64%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 3032b1dfe0..48a31127f4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 4cd5c664fd..0d0e38a1b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 96b72843ed..413d5ee9dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index c700c18dc0..5b94db28b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0541de7c63..1e1b352645 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index e3bb3a3c4b..f7912836c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 44dd058e1e..774377407a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbMsgCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 691a54aaab..671cd72262 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbRuleEngineConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 80d05628f2..75711d20b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -25,13 +25,13 @@ public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); +// private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); +// private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); +// private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); @@ -40,12 +40,12 @@ public class TbRuleEngineConsumerStats { if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } +// if (msg.hasPostTelemetry()) { +// postTelemetryCounter.incrementAndGet(); +// } +// if (msg.hasPostAttributes()) { +// postAttributesCounter.incrementAndGet(); +// } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -58,9 +58,9 @@ public class TbRuleEngineConsumerStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); - } +// if (msg.hasToServerRPCCallRequest()) { +// toServerRPCCallRequestCounter.incrementAndGet(); +// } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index d39465ff12..08578c3890 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; @@ -29,7 +28,6 @@ import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -45,12 +43,6 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TbKafkaSettings kafkaSettings; - @Value("${js.remote.request_topic}") private String requestTopic; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index c1b0608d44..fd8ee406c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; @@ -15,6 +30,8 @@ import org.thingsboard.server.provider.TbCoreQueueProvider; import java.util.UUID; import java.util.function.Consumer; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; + @Slf4j @Service @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") @@ -42,7 +59,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 980f5e6964..eed2b8fa8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -41,9 +41,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTranspo import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.provider.TbCoreQueueProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.TbCoreConsumerStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; @@ -65,7 +64,7 @@ import java.util.function.Consumer; @Slf4j @Service @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService implements RuleEngineTransportService { +public class RemoteRuleEngineTransportService { @Value("${transport.remote.rule_engine.topic}") private String ruleEngineTopic; @@ -85,12 +84,6 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Value("${transport.remote.rule_engine.stats.enabled:false}") private boolean statsEnabled; -// @Autowired -// private TbKafkaSettings kafkaSettings; -// - @Autowired - private TbNodeIdProvider nodeIdProvider; - @Autowired private ActorSystemContext actorContext; @@ -102,6 +95,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Autowired private DataDecodingEncodingService encodingService; + @Autowired + private TbCoreQueueProvider coreQueueProvider; + private TbQueueConsumer> ruleEngineConsumer; private TbQueueProducer> notificationsProducer; @@ -114,26 +110,12 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); - notificationsProducerBuilder.settings(kafkaSettings); - notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); - notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); - - notificationsProducer = notificationsProducerBuilder.build(); + notificationsProducer = coreQueueProvider.getTransportMsgProducer(); notificationsProducer.init(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); - ruleEngineConsumerBuilder.settings(kafkaSettings); - ruleEngineConsumerBuilder.topic(ruleEngineTopic); - ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - ruleEngineConsumerBuilder.groupId("tb-node"); - ruleEngineConsumerBuilder.autoCommit(true); - ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); - ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); - ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); - - ruleEngineConsumer = ruleEngineConsumerBuilder.build(); - ruleEngineConsumer.subscribe(); + //TODO: 2.5 +// ruleEngineConsumer = +// ruleEngineConsumer.subscribe(); } @EventListener(ApplicationReadyEvent.class) @@ -198,6 +180,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + //TODO: 2.5 id TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 354252642f..4765737430 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 3324f02440..5b11edb5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 7f8cff5f22..6917c23719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index 3d7d791ae4..e823d2a5fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ca51eb4ddb..ddf9d7d9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java similarity index 64% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java rename to common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java index 6e433e71a9..dc84625634 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java @@ -13,12 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server; -import java.util.UUID; +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; -public interface TbKafkaRequestIdExtractor { +@Data +@Component +public class TbQueueCoreSettings { - UUID extractRequestId(T value); + @Value("${queue.core.topic}") + private String topic; + @Value("${queue.core.partitions}") + private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 22714af77e..ca8f3a61da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index 9dad87588d..f95454c976 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java index 8eecae51fb..a3331999a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueMsgMetadata { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 41d165d743..206f753168 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -1,15 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; - public interface TbQueueProducer { void init(); String getDefaultTopic(); - ListenableFuture send(T msg, TbQueueCallback callback); + void send(T msg, TbQueueCallback callback); - ListenableFuture send(String topic, T msg, TbQueueCallback callback); + void send(String topic, T msg, TbQueueCallback callback); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5ba28f22c5..5182bb7260 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java index f3b4e4ad5d..686570ca32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java new file mode 100644 index 0000000000..603d787fa8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportApiSettings { + @Value("${queue.transport_api.requests_topic}") + private String requestsTopic; + + @Value("${queue.transport_api.responses_topic}") + private String responsesTopic; + + @Value("${queue.transport_api.max_pending_requests}") + private int maxPendingRequests; + + @Value("${queue.transport_api.max_requests_timeout}") + private int maxRequestsTimeout; + + @Value("${queue.transport_api.response_poll_interval}") + private long responsePollInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index 2f406a9631..b70fc417c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import java.nio.ByteBuffer; @@ -7,6 +22,7 @@ import java.util.UUID; public class AbstractTbQueueTemplate { protected static final String REQUEST_ID_HEADER = "requestId"; protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + protected static final String REQUEST_TIME = "requestTime"; protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); @@ -29,4 +45,17 @@ public class AbstractTbQueueTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } + + private static ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); + + protected static byte[] longToBytes(long x) { + longBuffer.putLong(0, x); + return longBuffer.array(); + } + + protected static long bytesToLong(byte[] bytes) { + longBuffer.put(bytes, 0, bytes.length); + longBuffer.flip();//need flip + return longBuffer.getLong(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index e547a6c070..de0c368a65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import org.thingsboard.server.TbQueueMsgHeaders; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 49022ad2e3..d022dafadc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import com.google.common.util.concurrent.Futures; @@ -79,7 +94,7 @@ public class DefaultTbQueueRequestTemplate { log.trace("Received response to Kafka Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId = null; + UUID requestId; if (requestIdHeader == null) { log.error("[{}] Missing requestId in header and body", response); } else { @@ -140,6 +155,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 92dac0551b..3804fb5160 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Builder; @@ -70,41 +85,46 @@ public class DefaultTbQueueResponseTemplate requests = requestTemplate.poll(pollInterval); + requests.forEach(request -> { - byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader); - String responseTopic = bytesToString(responseTopicHeader); - try { - pendingRequestCount.getAndIncrement(); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), - response -> { - pendingRequestCount.decrementAndGet(); - response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(responseTopic, response, null); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timeout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); + long currentTime = System.currentTimeMillis(); + long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); + if (requestTime + requestTimeout >= currentTime) { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(responseTopic, response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } } }); requestTemplate.commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index c072dc4d67..8dd492504c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Data; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java deleted file mode 100644 index 0c68c8dd5d..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.extern.slf4j.Slf4j; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public abstract class AbstractTbKafkaTemplate { - protected byte[] uuidToBytes(UUID uuid) { - ByteBuffer buf = ByteBuffer.allocate(16); - buf.putLong(uuid.getMostSignificantBits()); - buf.putLong(uuid.getLeastSignificantBits()); - return buf.array(); - } - - protected static UUID bytesToUuid(byte[] bytes) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - return new UUID(firstLong, secondLong); - } - - protected byte[] stringToBytes(String string) { - return string.getBytes(StandardCharsets.UTF_8); - } - - protected String bytesToString(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java deleted file mode 100644 index 17599bfccb..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import java.util.concurrent.Executor; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 05.10.18. - */ -public class AsyncCallbackTemplate { - - public static void withCallbackAndTimeout(ListenableFuture future, - Consumer onSuccess, - Consumer onFailure, - long timeoutInMs, - ScheduledExecutorService timeoutExecutor, - Executor callbackExecutor) { - future = Futures.withTimeout(future, timeoutInMs, TimeUnit.MILLISECONDS, timeoutExecutor); - withCallback(future, onSuccess, onFailure, callbackExecutor); - } - - public static void withCallback(ListenableFuture future, Consumer onSuccess, - Consumer onFailure, Executor executor) { - FutureCallback callback = new FutureCallback() { - @Override - public void onSuccess(T result) { - try { - onSuccess.accept(result); - } catch (Throwable th) { - onFailure(th); - } - } - - @Override - public void onFailure(Throwable t) { - onFailure.accept(t); - } - }; - if (executor != null) { - Futures.addCallback(future, callback, executor); - } else { - Futures.addCallback(future, callback); - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java index e44a93a70d..9f8e73bdfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java index 4698c226d6..09cc292deb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import lombok.AllArgsConstructor; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 9c6d911d33..0bb3a75db6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -15,16 +15,17 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.JdkFutureAdapters; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.kafka.clients.admin.*; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; import org.thingsboard.server.TbQueueAdmin; -import java.time.Duration; import java.util.Collections; -import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -40,6 +41,19 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } + @Override + public ListenableFuture createTopicIfNotExists(String topic) { + + KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); + + ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); + + return Futures.transformAsync(topicFuture, topicDescription -> { + KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); + return JdkFutureAdapters.listenInPoolThread(resultFuture); + }); + } + public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { synchronized (this) { long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); @@ -54,7 +68,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { } } - public CreateTopicsResult createTopic(NewTopic topic){ + public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } @@ -67,10 +81,4 @@ public class TBKafkaAdmin implements TbQueueAdmin { return false; } } - - @Override - public ListenableFuture createTopicIfNotExists(String topic) { - - return null; - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 6f0ce7b55c..558d556d09 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.UUID; /** * Created by ashvayka on 24.09.18. @@ -42,15 +41,11 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; - @Builder.Default - private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); - @Getter private final String topic; @Builder private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - TbKafkaRequestIdExtractor requestIdExtractor, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords) { @@ -68,7 +63,6 @@ public class TBKafkaConsumerTemplate implements TbQueueCon } this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; - this.requestIdExtractor = requestIdExtractor; this.topic = topic; } @@ -105,22 +99,8 @@ public class TBKafkaConsumerTemplate implements TbQueueCon consumer.unsubscribe(); } -// public void subscribe() { -// consumer.subscribe(Collections.singletonList(topic)); -// } -// - -// -// public ConsumerRecords poll(Duration duration) { -// return consumer.poll(duration); -// } - public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); } - public UUID extractRequestId(T value) { - return requestIdExtractor.extractRequestId(value); - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 938b278207..d76a2dc625 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -15,9 +15,6 @@ */ package org.thingsboard.server.kafka; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.JdkFutureAdapters; -import com.google.common.util.concurrent.ListenableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -31,7 +28,6 @@ import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import java.util.List; @@ -91,12 +87,12 @@ public class TBKafkaProducerTemplate implements TbQueuePro } @Override - public ListenableFuture send(T msg, TbQueueCallback callback) { - return send(defaultTopic, msg, callback); + public void send(T msg, TbQueueCallback callback) { + send(defaultTopic, msg, callback); } @Override - public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + public void send(String topic, T msg, TbQueueCallback callback) { String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; @@ -111,42 +107,8 @@ public class TBKafkaProducerTemplate implements TbQueuePro callback.onFailure(exception); } }); - - return Futures.transform(JdkFutureAdapters.listenInPoolThread(result), metadata -> new KafkaTbQueueMsgMetadata(metadata)); } -// public Future send(String key, T value, Callback callback) { -// return send(key, value, null, callback); -// } -// -// public Future send(String key, T value, Iterable
headers, Callback callback) { -// return send(key, value, null, headers, callback); -// } -// -// public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { -// if (!StringUtils.isEmpty(this.defaultTopic)) { -// return send(this.defaultTopic, key, value, timestamp, headers, callback); -// } else { -// throw new RuntimeException("Failed to send message! Default topic is not specified!"); -// } -// } -// -// public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { -// return send(topic, key, value, null, headers, callback); -// } -// -// public Future send(String topic, String key, T value, Callback callback) { -// return send(topic, key, value, null, null, callback); -// } -// -// public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { -// byte[] data = encoder.encode(value); -// ProducerRecord record; -// Integer partition = getPartition(topic, key, value, data); -// record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); -// return producer.send(record, callback); -// } - private Integer getPartition(String topic, T value) { if (partitioner == null) { return null; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java deleted file mode 100644 index 2a7bca119f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaProducerTemplate requestTemplate; - private final TBKafkaConsumerTemplate responseTemplate; - private final ConcurrentMap> pendingRequests; - private final boolean internalExecutor; - private final ExecutorService executor; - private final long maxRequestTimeout; - private final long maxPendingRequests; - private final long pollInterval; - private volatile long tickTs = 0L; - private volatile long tickSize = 0L; - private volatile boolean stopped = false; - - @Builder - public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, - TBKafkaConsumerTemplate responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxRequestTimeout = maxRequestTimeout; - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - if (executor != null) { - internalExecutor = false; - this.executor = executor; - } else { - internalExecutor = true; - this.executor = Executors.newSingleThreadExecutor(); - } - } - - public void init() { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.requestTemplate.getSettings()); - CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) { - log.trace("[{}] Topic already exists. ", responseTemplate.getTopic()); - } else { - log.info("[{}] Failed to create topic: {}", responseTemplate.getTopic(), e.getMessage(), e); - throw new RuntimeException(e); - } - - } - this.requestTemplate.init(); - tickTs = System.currentTimeMillis(); - responseTemplate.subscribe(); - executor.submit(() -> { - long nextCleanupMs = 0L; - while (!stopped) { - try { - ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); - if (responses.count() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.count()); - } - responses.forEach(response -> { - log.trace("Received response to Kafka Template request: {}", response); - Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - Response decodedResponse = null; - UUID requestId = null; - if (requestIdHeader == null) { - try { - decodedResponse = responseTemplate.decode(response); - requestId = responseTemplate.extractRequestId(decodedResponse); - } catch (IOException e) { - log.error("Failed to decode response", e); - } - } else { - requestId = bytesToUuid(requestIdHeader.value()); - } - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", response); - } else { - log.trace("[{}] Response received", requestId); - ResponseMetaData expectedResponse = pendingRequests.remove(requestId); - if (expectedResponse == null) { - log.trace("[{}] Invalid or stale request", requestId); - } else { - try { - if (decodedResponse == null) { - decodedResponse = responseTemplate.decode(response); - } - expectedResponse.future.set(decodedResponse); - } catch (IOException e) { - expectedResponse.future.setException(e); - } - } - } - }); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); - staleRequest.future.setException(new TimeoutException()); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain responses from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new responses", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (internalExecutor) { - executor.shutdownNow(); - } - } - - public ListenableFuture post(String key, Request request) { - if (tickSize > maxPendingRequests) { - return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); - } - UUID requestId = UUID.randomUUID(); - List
headers = new ArrayList<>(2); - headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); - headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); - SettableFuture future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); - pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); - requestTemplate.send(key, request, headers, (metadata, exception) -> { - if (exception != null) { - log.trace("[{}] Failed to post the request", requestId, exception); - } else { - log.trace("[{}] Posted the request: {}", requestId, metadata); - } - }); - return future; - } - - private static class ResponseMetaData { - private final long expTime; - private final SettableFuture future; - - ResponseMetaData(long ts, SettableFuture future) { - this.expTime = ts; - this.future = future; - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java deleted file mode 100644 index 864182a552..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.time.Duration; -import java.util.Collections; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaConsumerTemplate requestTemplate; - private final TBKafkaProducerTemplate responseTemplate; - private final TbKafkaHandler handler; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, - TBKafkaProducerTemplate responseTemplate, - TbKafkaHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.handler = handler; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - public void init() { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); - requests.forEach(request -> { - Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader.value()); - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", request); - return; - } - Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - String responseTopic = bytesToString(responseTopicHeader.value()); - try { - pendingRequestCount.getAndIncrement(); - Request decodedRequest = requestTemplate.decode(request); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), - response -> { - pendingRequestCount.decrementAndGet(); - reply(requestId, responseTopic, response); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timedout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - }); - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - - private void reply(UUID requestId, String topic, Response response) { - responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 6446643a18..566766fee6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -35,22 +35,22 @@ public class TbKafkaSettings { static final String REQUEST_ID_HEADER = "requestId"; static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - @Value("${kafka.bootstrap.servers}") + @Value("${queue.kafka.bootstrap.servers}") private String servers; - @Value("${kafka.acks}") + @Value("${queue.kafka.acks}") private String acks; - @Value("${kafka.retries}") + @Value("${queue.queue.kafka.retries}") private int retries; - @Value("${kafka.batch.size}") + @Value("${queue.kafka.batch.size}") private int batchSize; - @Value("${kafka.linger.ms}") + @Value("${queue.kafka.linger.ms}") private long lingerMs; - @Value("${kafka.buffer.memory}") + @Value("${queue.kafka.buffer.memory}") private long bufferMemory; @Value("${kafka.other:#{null}}") diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index abf30667b5..ded4cd9810 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index a0a35eefb8..b8ddcff89c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 32b9399164..e71d579c7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -1,11 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; @Data @@ -15,6 +27,10 @@ public class InMemoryTbQueueProducer implements TbQueuePro private final String defaultTopic; + public InMemoryTbQueueProducer(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + @Override public void init() { @@ -26,20 +42,18 @@ public class InMemoryTbQueueProducer implements TbQueuePro } @Override - public ListenableFuture send(T msg, TbQueueCallback callback) { - return send(defaultTopic, msg, callback); + public void send(T msg, TbQueueCallback callback) { + send(defaultTopic, msg, callback); } @Override - public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + public void send(String topic, T msg, TbQueueCallback callback) { boolean result = storage.put(topic, msg); if (result) { callback.onSuccess(null); - return Futures.immediateCheckedFuture(null); } else { Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); callback.onFailure(e); - return Futures.immediateFailedFuture(e); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java new file mode 100644 index 0000000000..79f5dd7da0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.memory.InMemoryTbQueueProducer; + +@Slf4j +@Component +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='in-memory'") +public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbQueueCoreSettings coreSettings; + + public InMemoryTbCoreQueueProvider(TbQueueCoreSettings coreSettings) { + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java new file mode 100644 index 0000000000..789ea1cf5a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.memory.InMemoryTbQueueProducer; + +@Component +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='in-memory'") +@Slf4j +public class InMemoryTransportQueueProvider implements TransportQueueProvider { + + private final TbQueueTransportApiSettings transportApiSettings; + + public InMemoryTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings) { + this.transportApiSettings = transportApiSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return producer; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return producer; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + return consumer; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 2f9a9ad8f8..d830cff8fa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -1,42 +1,102 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && ${queue.type:null}'=='kafka'") -public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='kafka'") +public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + @Override - public TbQueueProducer> getTransportMsgProducer() { - return null; + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return null; + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return null; + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { - return null; + public TbQueueConsumer> getToCoreMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO: 2.5 +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); + return responseBuilder.build(); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> getTransportApiRequestConsumer() { return null; } @Override - public TbQueueProducer> getTransportApiResponseProducer() { - return null; + public TbQueueProducer> getTransportApiResponseProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 945888a45a..3b671a3adb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import lombok.extern.slf4j.Slf4j; @@ -6,30 +21,77 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && ${queue.type:null}'=='kafka'") +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='kafka'") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueTransportApiSettings transportApiSettings; + + public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueTransportApiSettings transportApiSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.transportApiSettings = transportApiSettings; + } + @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiSettings.getResponsesTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO: 2.5 +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.requestTemplate(requestBuilder.build()); + templateBuilder.responseTemplate(responseBuilder.build()); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); } @Override public TbQueueProducer> getRuleEngineMsgProducer() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + return requestBuilder.build(); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + return requestBuilder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index b7e210f0f6..6123fc0859 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -1,9 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueResponseTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index 14d8efd534..de69e1819a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index e4dc2ba1f1..cdf6b3f763 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From e50796d84efbe572bb7f191313791083f6d1ffd9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 13 Mar 2020 20:33:17 +0200 Subject: [PATCH 011/102] Service Discovery improvements --- .../queue/DefaultTbCoreConsumerService.java | 15 ++ .../DefaultTbRuleEngineConsumerService.java | 15 ++ .../server/service/queue/MsgPackCallback.java | 15 ++ .../service/queue/TbCoreConsumerService.java | 15 ++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../server/service/queue/TbMsgCallback.java | 15 ++ .../queue/TbRuleEngineConsumerService.java | 15 ++ .../DefaultTbCoreToTransportService.java | 15 ++ .../transport/RemoteTransportApiService.java | 2 +- .../transport/TbCoreToTransportService.java | 15 ++ .../src/main/resources/thingsboard.yml | 12 +- common/queue/pom.xml | 16 +- .../org/thingsboard/server/TbQueueAdmin.java | 15 ++ .../thingsboard/server/TbQueueCallback.java | 15 ++ .../thingsboard/server/TbQueueConsumer.java | 15 ++ .../org/thingsboard/server/TbQueueMsg.java | 15 ++ .../thingsboard/server/TbQueueMsgHeaders.java | 15 ++ .../server/TbQueueMsgMetadata.java | 15 ++ .../thingsboard/server/TbQueueProducer.java | 18 ++ .../server/TbQueueRequestTemplate.java | 15 ++ .../server/TbQueueResponseTemplate.java | 15 ++ .../common/AbstractTbQueueTemplate.java | 15 ++ .../common/DefaultTbQueueMsgHeaders.java | 15 ++ .../common/DefaultTbQueueRequestTemplate.java | 15 ++ .../DefaultTbQueueResponseTemplate.java | 15 ++ .../server/common/TbProtoQueueMsg.java | 15 ++ .../DefaultTbServiceInfoProvider.java | 90 +++++++ .../discovery/PartitionChangeEvent.java | 33 +++ .../discovery/PartitionDiscoveryService.java | 29 ++ .../server/discovery/ServiceType.java | 20 ++ .../discovery/TbServiceInfoProvider.java | 30 +++ .../server/discovery/TopicPartitionInfo.java | 26 ++ .../ZkPartitionDiscoveryService.java | 255 ++++++++++++++++++ .../server/kafka/KafkaTbQueueMsg.java | 15 ++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 15 ++ .../server/memory/InMemoryStorage.java | 15 ++ .../memory/InMemoryTbQueueConsumer.java | 15 ++ .../memory/InMemoryTbQueueProducer.java | 15 ++ .../provider/KafkaTbCoreQueueProvider.java | 15 ++ .../provider/KafkaTransportQueueProvider.java | 15 ++ .../server/provider/TbCoreQueueProvider.java | 15 ++ .../provider/TransportQueueProvider.java | 15 ++ common/queue/src/main/proto/transport.proto | 10 + .../service/DefaultTransportService.java | 2 +- 44 files changed, 981 insertions(+), 14 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 4cd5c664fd..0d0e38a1b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 96b72843ed..413d5ee9dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index c700c18dc0..5b94db28b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0541de7c63..1e1b352645 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index e3bb3a3c4b..f7912836c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 44dd058e1e..774377407a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbMsgCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 691a54aaab..671cd72262 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbRuleEngineConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index c1b0608d44..49031f9f53 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 354252642f..4765737430 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 3324f02440..5b11edb5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2ded608203..da2e5beb98 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -70,11 +70,6 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# RPC connection parameters. Used only in cluster mode only. -rpc: - bind_host: "${RPC_HOST:localhost}" - bind_port: "${RPC_PORT:9001}" - # Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: # Unique id for this node (autogenerated if empty) @@ -521,7 +516,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:Thingsboard REST API}" + title: "${SWAGGER_TITLE:ThingsBoard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -568,4 +563,7 @@ queue: topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport \ No newline at end of file + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/common/queue/pom.xml b/common/queue/pom.xml index fd26358cc6..429f68acb0 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + util + org.thingsboard.common message @@ -84,6 +88,14 @@ ch.qos.logback logback-classic + + com.google.protobuf + protobuf-java + + + org.apache.curator + curator-recipes + junit junit @@ -94,10 +106,6 @@ mockito-all test - - com.google.protobuf - protobuf-java - diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 7f8cff5f22..6917c23719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index 3d7d791ae4..e823d2a5fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ca51eb4ddb..ddf9d7d9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 22714af77e..ca8f3a61da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index 9dad87588d..f95454c976 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java index 8eecae51fb..a3331999a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueMsgMetadata { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 41d165d743..2ec01357b7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -1,6 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.discovery.TopicPartitionInfo; public interface TbQueueProducer { @@ -12,4 +28,6 @@ public interface TbQueueProducer { ListenableFuture send(String topic, T msg, TbQueueCallback callback); + ListenableFuture send(String topic, int partition, T msg, TbQueueCallback callback); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5ba28f22c5..5182bb7260 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java index f3b4e4ad5d..686570ca32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index 2f406a9631..118cf5e0b7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import java.nio.ByteBuffer; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index e547a6c070..de0c368a65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import org.thingsboard.server.TbQueueMsgHeaders; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 49022ad2e3..6876c2f3d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 92dac0551b..ba32b399ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Builder; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index c072dc4d67..8dd492504c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Data; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java new file mode 100644 index 0000000000..0b09a333d8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { + + @Getter + @Value("${service.id:#{null}}") + private String serviceId; + + @Getter + @Value("${service.type:monolith}") + private String serviceType; + + @Getter + @Value("${service.tenant_id:}") + private String tenantIdStr; + + private List serviceTypes; + private ServiceInfo serviceInfo; + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(serviceId)) { + try { + serviceId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current Service ID: {}", serviceId); + if (serviceType.equalsIgnoreCase("monolith")) { + serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); + } else { + serviceTypes = Collections.singletonList(ServiceType.valueOf(serviceType)); + } + ServiceInfo.Builder builder = ServiceInfo.newBuilder() + .setServiceId(serviceId) + .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + if (!StringUtils.isEmpty(tenantIdStr)) { + UUID tenantId = UUID.fromString(tenantIdStr); + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + } + serviceInfo = builder.build(); + } + + + @Override + public List getSupportedServiceTypes() { + return serviceTypes; + } + + @Override + public ServiceInfo getServiceInfo() { + return serviceInfo; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java new file mode 100644 index 0000000000..7d083ba873 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + + +public class PartitionChangeEvent extends ApplicationEvent { + + @Getter + private final List partitions; + + public PartitionChangeEvent(Object source, List partitions) { + super(source); + this.partitions = partitions; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java new file mode 100644 index 0000000000..d85b1d3f44 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +public interface PartitionDiscoveryService { + + List getCurrentPartitions(ServiceType serviceType); + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java new file mode 100644 index 0000000000..801069eb61 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +public enum ServiceType { + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java new file mode 100644 index 0000000000..5cb545c5d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import java.util.List; + +public interface TbServiceInfoProvider { + + List getSupportedServiceTypes(); + + String getServiceId(); + + ServiceInfo getServiceInfo(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java new file mode 100644 index 0000000000..b46b941360 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Data; + +@Data +public class TopicPartitionInfo { + + private String topic; + private int partition; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java new file mode 100644 index 0000000000..a722f2f395 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -0,0 +1,255 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.curator.retry.RetryForever; +import org.apache.curator.utils.CloseableUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) +@Slf4j +public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, PathChildrenCacheListener { + + @Value("${zk.url}") + private String zkUrl; + @Value("${zk.retry_interval_ms}") + private Integer zkRetryInterval; + @Value("${zk.connection_timeout_ms}") + private Integer zkConnectionTimeout; + @Value("${zk.session_timeout_ms}") + private Integer zkSessionTimeout; + @Value("${zk.zk_dir}") + private String zkDir; + + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.rule_engine.partitions:100}") + private Integer ruleEnginePartitions; + + @Autowired + private TbServiceInfoProvider serviceIdProvider; + + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + + private ExecutorService reconnectExecutorService; + private CuratorFramework client; + private PathChildrenCache cache; + private String nodePath; + private String zkNodesDir; + + private volatile boolean stopped = true; + + @Override + public List getCurrentPartitions(ServiceType serviceType) { + return Collections.emptyList(); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + + } + + @PostConstruct + public void init() { + log.info("Initializing..."); + Assert.hasLength(zkUrl, missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); + + reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); + + partitionSizes.put(ServiceType.TB_CORE, corePartitions); + partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); + + log.info("Initializing discovery service using ZK connect string: {}", zkUrl); + + zkNodesDir = zkDir + "/nodes"; + initZkClient(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } else { + log.info("Received application ready event. Starting current ZK node."); + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + getOtherServers().forEach( + server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) + ); + } + + @Override + public synchronized void publishCurrentServer() { + ServerInstance self = this.serverInstance.getSelf(); + if (currentServerExists()) { + log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + } else { + try { + log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + nodePath = client.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); + log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + client.getConnectionStateListenable().addListener(checkReconnect(self)); + } catch (Exception e) { + log.error("Failed to create ZK node", e); + throw new RuntimeException(e); + } + } + } + + private boolean currentServerExists() { + if (nodePath == null) { + return false; + } + try { + ServerInstance self = this.serverInstance.getSelf(); + ServerAddress registeredServerAdress = null; + registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); + if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + return true; + } + } catch (KeeperException.NoNodeException e) { + log.info("ZK node does not exist: {}", nodePath); + } catch (Exception e) { + log.error("Couldn't check if ZK node exists", e); + } + return false; + } + + private ConnectionStateListener checkReconnect(ServerInstance self) { + return (client, newState) -> { + log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + if (newState == ConnectionState.LOST) { + reconnectExecutorService.submit(this::reconnect); + } + }; + } + + private volatile boolean reconnectInProgress = false; + + private synchronized void reconnect() { + if (!reconnectInProgress) { + reconnectInProgress = true; + try { + destroyZkClient(); + initZkClient(); + publishCurrentServer(); + } catch (Exception e) { + log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); + } finally { + reconnectInProgress = false; + } + } + } + + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } + } + + private void unpublishCurrentServer() { + try { + if (nodePath != null) { + client.delete().forPath(nodePath); + } + } catch (Exception e) { + log.error("Failed to delete ZK node {}", nodePath, e); + throw new RuntimeException(e); + } + } + + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) { + } + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); + } + + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); + } + + public static String missingProperty(String propertyName) { + return "The " + propertyName + " property need to be set!"; + } + + @Override + public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { + + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java index e44a93a70d..9f8e73bdfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java index 4698c226d6..09cc292deb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import lombok.AllArgsConstructor; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index abf30667b5..ded4cd9810 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index a0a35eefb8..b8ddcff89c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 32b9399164..c16d34a81a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 2f9a9ad8f8..62bc2dd5ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 945888a45a..265373793a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index b7e210f0f6..43878687ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index 14d8efd534..de69e1819a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto index 892139a84a..8c564b6540 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -19,6 +19,16 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +/** + * Service Discovery Data Structures; + */ +message ServiceInfo { + string serviceId = 1; + repeated string serviceTypes = 2; + int64 tenantIdMSB = 3; + int64 tenantIdLSB = 4; +} + /** * Transport Service Data Structures; */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index e4dc2ba1f1..cdf6b3f763 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 23c017567bf59022bef311c9b2d641fc3c31c7a4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 16 Mar 2020 19:24:56 +0200 Subject: [PATCH 012/102] Implementation of Queue Interfaces --- .../server/actors/ActorSystemContext.java | 21 +- .../server/actors/app/AppActor.java | 14 +- .../device/DeviceActorMessageProcessor.java | 2 +- .../actors/rpc/BasicRpcSessionListener.java | 83 ----- .../server/actors/rpc/RpcBroadcastMsg.java | 27 -- .../server/actors/rpc/RpcManagerActor.java | 230 ------------ .../server/actors/rpc/RpcSessionActor.java | 135 ------- .../actors/rpc/RpcSessionClosedMsg.java | 29 -- .../actors/rpc/RpcSessionConnectedMsg.java | 31 -- .../rpc/RpcSessionCreateRequestMsg.java | 35 -- .../actors/rpc/RpcSessionDisconnectedMsg.java | 29 -- .../server/actors/rpc/RpcSessionTellMsg.java | 27 -- .../server/actors/rpc/SessionActorInfo.java | 30 -- .../actors/ruleChain/DefaultTbContext.java | 21 +- .../RuleChainActorMessageProcessor.java | 15 +- .../server/actors/service/ActorService.java | 4 +- .../actors/service/DefaultActorService.java | 213 +++++------ .../server/actors/stats/StatsActor.java | 3 +- .../CurrentServerInstanceService.java | 55 --- .../cluster/discovery/DiscoveryService.java | 33 -- .../discovery/DiscoveryServiceListener.java | 28 -- .../discovery/DummyDiscoveryService.java | 67 ---- .../cluster/discovery/ServerInstance.java | 47 --- .../discovery/ServerInstanceService.java | 24 -- .../cluster/discovery/ZkDiscoveryService.java | 330 ------------------ .../routing/ClusterRoutingService.java | 35 -- .../ConsistentClusterRoutingService.java | 153 -------- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 40 ++- .../queue/TbRuleEngineConsumerStats.java | 59 +--- .../service/rpc/DefaultDeviceRpcService.java | 29 +- .../state/DefaultDeviceStateService.java | 91 ++--- .../DefaultTelemetrySubscriptionService.java | 38 +- .../BaseRuleChainTransactionService.java | 25 +- .../DefaultTbCoreToTransportService.java | 9 +- .../RemoteRuleEngineTransportService.java | 237 ------------- .../transport/RemoteTransportApiService.java | 3 +- .../transport/TransportApiRequestDecoder.java | 31 -- .../TransportApiResponseEncoder.java | 30 -- .../src/main/resources/thingsboard.yml | 3 +- .../ConsistentClusterRoutingServiceTest.java | 71 ++-- .../thingsboard/server/TbQueueProducer.java | 10 +- .../common/DefaultTbQueueRequestTemplate.java | 5 +- .../DefaultTbQueueResponseTemplate.java | 3 +- .../discovery}/ConsistentHashCircle.java | 18 +- .../ConsistentHashPartitionService.java | 221 ++++++++++++ .../DefaultTbServiceInfoProvider.java | 11 +- .../discovery/DummyDiscoveryService.java | 32 ++ .../discovery/PartitionDiscoveryService.java | 9 - .../server/discovery/PartitionService.java | 16 + .../server/discovery/TopicPartitionInfo.java | 23 +- .../ZkPartitionDiscoveryService.java | 122 ++++--- .../environment/EnvironmentLogService.java | 4 +- .../server/kafka/TBKafkaProducerTemplate.java | 50 +-- .../memory/InMemoryTbQueueProducer.java | 17 +- ...ava => InMemoryMonolithQueueProvider.java} | 12 +- .../provider/KafkaMonolithQueueProvider.java | 119 +++++++ .../provider/KafkaTbCoreQueueProvider.java | 33 +- .../KafkaTbRuleEngineQueueProvider.java | 86 +++++ .../provider/KafkaTransportQueueProvider.java | 8 +- .../provider/TbRuleEngineQueueProvider.java | 61 ++++ common/transport/transport-api/pom.xml | 4 + .../service/DefaultTransportService.java | 38 +- 63 files changed, 1055 insertions(+), 2236 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java rename {application/src/main/java/org/thingsboard/server/service/cluster/routing => common/queue/src/main/java/org/thingsboard/server/discovery}/ConsistentHashCircle.java (66%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java rename {application/src/main/java/org/thingsboard/server/service => common/queue/src/main/java/org/thingsboard/server}/environment/EnvironmentLogService.java (90%) rename common/queue/src/main/java/org/thingsboard/server/provider/{InMemoryTbCoreQueueProvider.java => InMemoryMonolithQueueProvider.java} (85%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index d10436075b..e5a0e6a25a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -65,8 +65,6 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -108,19 +106,11 @@ public class ActorSystemContext { @Setter private ActorService actorService; - @Autowired - @Getter - private DiscoveryService discoveryService; - @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - @Autowired @Getter private DataDecodingEncodingService encodingService; @@ -368,7 +358,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); persistEvent(event); } @@ -377,7 +368,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -406,8 +398,11 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); } + public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + //TODO 2.5 +// return discoveryService.getCurrentServer().getServerAddress().toString(); + return null; } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..06f936a9e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -134,13 +134,15 @@ public class AppActor extends RuleChainManagerActor { } private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); +// if (address.isPresent()) { + +// systemContext.getRpcService().tell( +// systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); +// } else { self().tell(msg.getMsg(), ActorRef.noSender()); - } +// } } private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6c0dde4044..6b878ef7d8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -260,7 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - //TODO: 2.5 move this as a notification to the queue; + //TODO 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java deleted file mode 100644 index f0ff8ce9e1..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcBroadcastMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java deleted file mode 100644 index bb87d2ea88..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionClosedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java deleted file mode 100644 index 489a5bfa17..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionConnectedMsg { - - private final ServerAddress remoteAddress; - private final UUID id; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java deleted file mode 100644 index df41a7e95f..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.stub.StreamObserver; -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionCreateRequestMsg { - - private final UUID msgUid; - private final ServerAddress remoteAddress; - private final StreamObserver responseObserver; - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java deleted file mode 100644 index 14e7504637..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionDisconnectedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java deleted file mode 100644 index 3832d6eb94..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionTellMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java deleted file mode 100644 index 811713819a..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.Data; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class SessionActorInfo { - protected final UUID sessionId; - protected final ActorRef actor; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..f5ea5c399f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -121,8 +121,10 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + //TODO 2.5 +// Optional address = mainCtx.getRoutingService().resolveById(entityId); +// return !address.isPresent(); + return true; } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -353,13 +355,14 @@ class DefaultTbContext implements TbContext { src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); + //TODO 2.5 +// ServerAddress requestOriginAddress; +// if (!StringUtils.isEmpty(src.getOriginHost())) { +// requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); +// } else { +// requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); +// } +// mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); } consumer.accept(RuleEngineDeviceRpcResponse.builder() .deviceId(src.getDeviceId()) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 49d019c1dc..095af7f62d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -230,20 +230,21 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); - - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(originatorEntityId); +// if (address.isPresent()) { +// onRemoteTellNext(address.get(), envelope); +// } else { onLocalTellNext(envelope); - } +// } } private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { TbMsg msg = envelope.getMsg(); log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + //TODO 2.5 +// systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); } private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 48a31127f4..890128e645 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -21,10 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { +public interface ActorService extends SessionMsgProcessor { void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index e5da93b9e2..fc9eb537c1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -19,7 +19,6 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; import akka.actor.Props; import akka.actor.Terminated; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -32,25 +31,16 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.app.AppActor; import org.thingsboard.server.actors.app.AppInitMsg; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcManagerActor; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DeviceStateService; import scala.concurrent.Await; import scala.concurrent.Future; @@ -60,8 +50,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - @Service @Slf4j public class DefaultActorService implements ActorService { @@ -77,12 +65,6 @@ public class DefaultActorService implements ActorService { @Autowired private ActorSystemContext actorContext; - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DiscoveryService discoveryService; - @Autowired private DeviceStateService deviceStateService; @@ -102,13 +84,9 @@ public class DefaultActorService implements ActorService { appActor = system.actorOf(Props.create(new AppActor.ActorCreator(actorContext)).withDispatcher(APP_DISPATCHER_NAME), "appActor"); actorContext.setAppActor(appActor); - rpcManagerActor = system.actorOf(Props.create(new RpcManagerActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), - "rpcManagerActor"); - ActorRef statsActor = system.actorOf(Props.create(new StatsActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), "statsActor"); actorContext.setStatsActor(statsActor); - rpcService.init(this); log.info("Actor system initialized."); } @@ -134,22 +112,23 @@ public class DefaultActorService implements ActorService { appActor.tell(msg, ActorRef.noSender()); } - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } + //TODO 2.5 +// @Override +// public void onServerAdded(ServerInstance server) { +// log.trace("Processing onServerAdded msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), true)); +// } +// +// @Override +// public void onServerUpdated(ServerInstance server) { +// //Do nothing +// } +// +// @Override +// public void onServerRemoved(ServerInstance server) { +// log.trace("Processing onServerRemoved msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), false)); +// } @Override public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { @@ -172,12 +151,13 @@ public class DefaultActorService implements ActorService { public void broadcast(ToAllNodesMsg msg) { actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); + //TODO 2.5 +// rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage +// .newBuilder() +// .setPayload(ByteString +// .copyFrom(actorContext.getEncodingService().encode(msg))) +// .setMessageType(CLUSTER_ACTOR_MESSAGE) +// .build())); appActor.tell(msg, ActorRef.noSender()); } @@ -204,79 +184,78 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } + //TODO 2.5 +// @Override +// public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// receivedClusterMsgs.incrementAndGet(); +// } +// ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); +// if (log.isDebugEnabled()) { +// log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); +// log.info("MSG: {}", msg); +// } +// switch (msg.getMessageType()) { +// case CLUSTER_ACTOR_MESSAGE: +// java.util.Optional decodedMsg = actorContext.getEncodingService() +// .decode(msg.getPayload().toByteArray()); +// if (decodedMsg.isPresent()) { +// appActor.tell(decodedMsg.get(), ActorRef.noSender()); +// } else { +// log.error("Error during decoding cluster proto message"); +// } +// break; +// case TO_ALL_NODES_MSG: +// //TODO +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: +// actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: +// actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: +// actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TRANSACTION_SERVICE_MESSAGE: +// actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// } +// } +// @Override +// public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// +// @Override +// public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// @Override +// public void onBroadcastMsg(RpcBroadcastMsg msg) { +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } @Override public void onDeviceAdded(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..c8e55bf9ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -58,7 +58,8 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + //TODO 2.5 +// event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java deleted file mode 100644 index d0ba27814c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import java.util.List; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryService { - - void publishCurrentServer(); - - void unpublishCurrentServer(); - - ServerInstance getCurrentServer(); - - List getOtherServers(); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java deleted file mode 100644 index ef6b565817..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryServiceListener { - - void onServerAdded(ServerInstance server); - - void onServerUpdated(ServerInstance server); - - void onServerRemoved(ServerInstance server); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java deleted file mode 100644 index 009a7801e7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.DependsOn; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.Collections; -import java.util.List; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -@Slf4j -@DependsOn("environmentLogService") -public class DummyDiscoveryService implements DiscoveryService { - - @Autowired - private ServerInstanceService serverInstance; - - @PostConstruct - public void init() { - log.info("Initializing..."); - } - - @Override - public void publishCurrentServer() { - //Do nothing - } - - @Override - public void unpublishCurrentServer() { - //Do nothing - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return Collections.emptyList(); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java deleted file mode 100644 index 095e10f0b3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { - - ServerInstance getSelf(); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java deleted file mode 100644 index e1cdd5f83e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.SerializationException; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.RetryForever; -import org.apache.curator.utils.CloseableUtils; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) -@Slf4j -public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener { - - @Value("${zk.url}") - private String zkUrl; - @Value("${zk.retry_interval_ms}") - private Integer zkRetryInterval; - @Value("${zk.connection_timeout_ms}") - private Integer zkConnectionTimeout; - @Value("${zk.session_timeout_ms}") - private Integer zkSessionTimeout; - @Value("${zk.zk_dir}") - private String zkDir; - - private String zkNodesDir; - - @Autowired - private ServerInstanceService serverInstance; - - @Autowired - @Lazy - private TelemetrySubscriptionService tsSubService; - - @Autowired - @Lazy - private DeviceStateService deviceStateService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - @Lazy - private ClusterRoutingService routingService; - - private ExecutorService reconnectExecutorService; - - private CuratorFramework client; - private PathChildrenCache cache; - private String nodePath; - - private volatile boolean stopped = true; - - @PostConstruct - public void init() { - log.info("Initializing..."); - Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); - - reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); - - zkNodesDir = zkDir + "/nodes"; - initZkClient(); - } - - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) {} - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); - } - - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); - } - - @Override - public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); - if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); - } else { - try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); - nodePath = client.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); - client.getConnectionStateListenable().addListener(checkReconnect(self)); - } catch (Exception e) { - log.error("Failed to create ZK node", e); - throw new RuntimeException(e); - } - } - } - - private boolean currentServerExists() { - if (nodePath == null) { - return false; - } - try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { - return true; - } - } catch (KeeperException.NoNodeException e) { - log.info("ZK node does not exist: {}", nodePath); - } catch (Exception e) { - log.error("Couldn't check if ZK node exists", e); - } - return false; - } - - private ConnectionStateListener checkReconnect(ServerInstance self) { - return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); - if (newState == ConnectionState.LOST) { - reconnectExecutorService.submit(this::reconnect); - } - }; - } - - private volatile boolean reconnectInProgress = false; - - private synchronized void reconnect() { - if (!reconnectInProgress) { - reconnectInProgress = true; - try { - destroyZkClient(); - initZkClient(); - publishCurrentServer(); - } catch (Exception e) { - log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); - } finally { - reconnectInProgress = false; - } - } - } - - @Override - public void unpublishCurrentServer() { - try { - if (nodePath != null) { - client.delete().forPath(nodePath); - } - } catch (Exception e) { - log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); - } - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); - } catch (NoSuchElementException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting current ZK node."); - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); - } - - @Override - public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - if (stopped) { - log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); - return; - } - ChildData data = pathChildrenCacheEvent.getData(); - if (data == null) { - log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); - return; - } else if (data.getData() == null) { - log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); - return; - } else if (nodePath != null && nodePath.equals(data.getPath())) { - if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { - log.info("ZK node for current instance is somehow deleted."); - publishCurrentServer(); - } - log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); - return; - } - ServerInstance instance; - try { - ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); - instance = new ServerInstance(serverAddress); - } catch (SerializationException e) { - log.error("Failed to decode server instance for node {}", data.getPath(), e); - throw e; - } - log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); - switch (pathChildrenCacheEvent.getType()) { - case CHILD_ADDED: - routingService.onServerAdded(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerAdded(instance); - break; - case CHILD_UPDATED: - routingService.onServerUpdated(instance); - actorService.onServerUpdated(instance); - break; - case CHILD_REMOVED: - routingService.onServerRemoved(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerRemoved(instance); - break; - default: - break; - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java deleted file mode 100644 index e0ed64fdbd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; - -import java.util.Optional; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { - - ServerAddress getCurrentServer(); - - Optional resolveById(EntityId entityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 0d0e38a1b2..295ce1aa13 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -45,7 +45,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") @Slf4j public class DefaultTbCoreConsumerService implements TbCoreConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 413d5ee9dd..ef2b44bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -44,26 +45,26 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") @Slf4j public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { - @Value("${queue.rule-engine.poll_interval}") + @Value("${queue.rule_engine.poll_interval}") private long pollDuration; - @Value("${queue.rule-engine.pack_processing_timeout}") + @Value("${queue.rule_engine.pack_processing_timeout}") private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:false}") + @Value("${queue.rule_engine.stats.enabled:false}") private boolean statsEnabled; private final ActorSystemContext actorContext; - private final TbQueueConsumer> consumer; + private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { - this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + public DefaultTbRuleEngineConsumerService(TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); this.actorContext = actorContext; } @@ -78,17 +79,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); - ConcurrentMap> ackMap = msgs.stream().collect( + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); ackMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { - TransportProtos.ToCoreMsg toCoreMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toCoreMsg); - if (toCoreMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToRuleEngineMsg()) { + forwardToRuleEngineActor(toRuleEngineMsg.getToRuleEngineMsg(), callback); } else { callback.onSuccess(); } @@ -112,11 +113,12 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } - private void forwardToDeviceActor(TransportProtos.TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + //TODO 2.5 + private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toDeviceActorMsg, TbMsgCallback callback) { +// if (statsEnabled) { +// stats.log(toDeviceActorMsg); +// } +// actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); } @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 75711d20b2..a5e8ba94af 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,60 +24,29 @@ import java.util.concurrent.atomic.AtomicInteger; public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); - private final AtomicInteger sessionEventCounter = new AtomicInteger(0); -// private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); -// private final AtomicInteger postAttributesCounter = new AtomicInteger(0); - private final AtomicInteger getAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); - private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); -// private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); - private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); + private final AtomicInteger postAttributesCounter = new AtomicInteger(0); + private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - public void log(TransportProtos.TransportToDeviceActorMsg msg) { + public void log(TransportProtos.TransportToRuleEngineMsg msg) { totalCounter.incrementAndGet(); - if (msg.hasSessionEvent()) { - sessionEventCounter.incrementAndGet(); + if (msg.hasPostTelemetry()) { + postTelemetryCounter.incrementAndGet(); } -// if (msg.hasPostTelemetry()) { -// postTelemetryCounter.incrementAndGet(); -// } -// if (msg.hasPostAttributes()) { -// postAttributesCounter.incrementAndGet(); -// } - if (msg.hasGetAttributes()) { - getAttributesCounter.incrementAndGet(); + if (msg.hasPostAttributes()) { + postAttributesCounter.incrementAndGet(); } - if (msg.hasSubscribeToAttributes()) { - subscribeToAttributesCounter.incrementAndGet(); - } - if (msg.hasSubscribeToRPC()) { - subscribeToRPCCounter.incrementAndGet(); - } - if (msg.hasToDeviceRPCCallResponse()) { - toDeviceRPCCallResponseCounter.incrementAndGet(); - } -// if (msg.hasToServerRPCCallRequest()) { -// toServerRPCCallRequestCounter.incrementAndGet(); -// } - if (msg.hasSubscriptionInfo()) { - subscriptionInfoCounter.incrementAndGet(); - } - if (msg.hasClaimDevice()) { - claimDeviceCounter.incrementAndGet(); + if (msg.hasToServerRPCCallRequest()) { + toServerRPCCallRequestCounter.incrementAndGet(); } } public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), - subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + log.info("Transport total [{}] telemetry [{}] attributes [{}] toServerRpc [{}]", + total, postTelemetryCounter.getAndSet(0), + postAttributesCounter.getAndSet(0), toServerRPCCallRequestCounter.getAndSet(0)); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java index 5d303b54ab..8baecbf37a 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -42,8 +42,6 @@ import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -64,12 +62,6 @@ public class DefaultDeviceRpcService implements DeviceRpcService { private static final ObjectMapper json = new ObjectMapper(); - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private DeviceService deviceService; @@ -106,7 +98,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { @Override public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { + //TODO 2.5 + if (true) {//routingService.getCurrentServer().equals(requestOriginAddress) UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -124,7 +117,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } else { builder.setError(-1); } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); } } @@ -159,7 +153,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); + //TODO 2.5 +// processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); } @Override @@ -172,8 +167,9 @@ public class DefaultDeviceRpcService implements DeviceRpcService { ObjectNode entityNode = json.createObjectNode(); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); + //TODO 2.5 +// metaData.putValue("originHost", routingService.getCurrentServer().getHost()); +// metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); metaData.putValue("oneway", Boolean.toString(msg.isOneway())); @@ -197,9 +193,10 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); + //TODO 2.5 +// ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); +// log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); +// forward(msg.getDeviceId(), rpcMsg); } private void forward(DeviceId deviceId, T msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index f40de180ab..31b712644d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -57,21 +57,31 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.ACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.CONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.DISCONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.INACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; /** * Created by ashvayka on 01.05.18. @@ -111,12 +121,6 @@ public class DefaultDeviceStateService implements DeviceStateService { @Autowired private TelemetrySubscriptionService tsSubService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -235,19 +239,20 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { if (!deviceStates.containsKey(device.getId())) { fetchFutures.add(fetchDeviceState(device)); } - } else { - Set tenantDeviceSet = tenantDevices.get(tenant.getId()); - if (tenantDeviceSet != null) { - tenantDeviceSet.remove(device.getId()); - } - deviceStates.remove(device.getId()); - deviceLastReportedActivity.remove(device.getId()); - deviceLastSavedActivity.remove(device.getId()); - } +// } else { +// Set tenantDeviceSet = tenantDevices.get(tenant.getId()); +// if (tenantDeviceSet != null) { +// tenantDeviceSet.remove(device.getId()); +// } +// deviceStates.remove(device.getId()); +// deviceLastReportedActivity.remove(device.getId()); +// deviceLastSavedActivity.remove(device.getId()); +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -268,9 +273,10 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { fetchFutures.add(fetchDeviceState(device)); - } +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -356,7 +362,8 @@ public class DefaultDeviceStateService implements DeviceStateService { private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - if (!routingService.resolveById(deviceId).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(deviceId).isPresent()) { Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); if (device != null) { try { @@ -366,7 +373,7 @@ public class DefaultDeviceStateService implements DeviceStateService { log.debug("[{}] Failed to fetch device state!", deviceId, e); } } - } +// } } return deviceStateData; } @@ -389,8 +396,9 @@ public class DefaultDeviceStateService implements DeviceStateService { } private void onDeviceAddedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { Futures.addCallback(fetchDeviceState(device), new FutureCallback() { @Override public void onSuccess(@Nullable DeviceStateData state) { @@ -402,9 +410,9 @@ public class DefaultDeviceStateService implements DeviceStateService { log.warn("Failed to register device to the state service", t); } }); - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); +// } } private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { @@ -417,12 +425,14 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); } private void onDeviceUpdatedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); if (stateData != null) { TbMsgMetaData md = new TbMsgMetaData(); @@ -430,14 +440,15 @@ public class DefaultDeviceStateService implements DeviceStateService { md.putValue("deviceType", device.getType()); stateData.setMetaData(md); } - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); +// } } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - Optional address = routingService.resolveById(deviceId); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(deviceId); +// if (!address.isPresent()) { deviceStates.remove(deviceId); deviceLastReportedActivity.remove(deviceId); deviceLastSavedActivity.remove(deviceId); @@ -448,9 +459,9 @@ public class DefaultDeviceStateService implements DeviceStateService { tenantDevices.remove(tenantId); } } - } else { - sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); - } +// } else { +// sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); +// } } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d29e5de5b7..04952a3bd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -56,8 +56,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.Subscription; @@ -102,12 +100,6 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio @Autowired private TimeseriesService tsService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private EntityViewService entityViewService; @@ -152,7 +144,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio endTime = entityView.getEndTimeMs(); sub = getUpdatedSubscriptionState(entityId, sub, entityView); } - Optional server = routingService.resolveById(entityId); + //TODO 2.5 + Optional server = Optional.empty();//routingService.resolveById(entityId); Subscription subscription; if (server.isPresent()) { ServerAddress address = server.get(); @@ -340,7 +333,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio while (deviceIterator.hasNext()) { Map.Entry> e = deviceIterator.next(); Set subscriptions = e.getValue(); - Optional newAddressOptional = routingService.resolveById(e.getKey()); + //TODO 2.5 + Optional newAddressOptional = Optional.empty();// routingService.resolveById(e.getKey()); if (newAddressOptional.isPresent()) { newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); } else { @@ -424,7 +418,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalAttributesUpdate(entityId, scope, attributes); if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { @@ -440,7 +435,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onTimeseriesUpdate(EntityId entityId, List ts) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalTimeseriesUpdate(entityId, ts); } else { @@ -632,7 +628,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { @@ -657,7 +654,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.addData(dataBuilder.build()); } ); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { @@ -666,7 +664,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityType(entityId.getEntityType().name()); builder.setScope(scope); attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { @@ -674,17 +673,20 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityId(entityId.getId().toString()); builder.setEntityType(entityId.getEntityType().name()); ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSessionClose(ServerAddress address, String sessionId) { ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); } private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); } private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 1aee532c93..173c293363 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -24,9 +24,6 @@ import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import javax.annotation.PostConstruct; @@ -50,12 +47,6 @@ import java.util.function.Consumer; @Slf4j public class BaseRuleChainTransactionService implements RuleChainTransactionService { - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Autowired private DbCallbackExecutorService callbackExecutor; @@ -110,13 +101,14 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ @Override public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); - if (address.isPresent()) { - sendTransactionEventToRemoteServer(msg, address.get()); - executeOnSuccess(onSuccess, msg); - } else { + //TODO 2.5 +// Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); +// if (address.isPresent()) { +// sendTransactionEventToRemoteServer(msg, address.get()); +// executeOnSuccess(onSuccess, msg); +// } else { endLocalTransaction(msg, onSuccess, onFailure); - } +// } } @Override @@ -237,6 +229,7 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index fd8ee406c2..f7afd105dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; @@ -34,17 +35,15 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Slf4j @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public class DefaultTbCoreToTransportService implements TbCoreToTransportService { - private final TbCoreQueueProvider tbCoreQueueProvider; private final TbQueueProducer> tbTransportProducer; @Value("${queue.notifications.topic}") private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbCoreQueueProvider = tbCoreQueueProvider; this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); } @@ -60,7 +59,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); - tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + tbTransportProducer.send(TopicPartitionInfo.builder().topic(topic).build(), queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private static class QueueCallbackAdaptor implements TbQueueCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java deleted file mode 100644 index eed2b8fa8b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.BlockingBucket; -import io.github.bucket4j.Bucket4j; -import io.github.bucket4j.local.LocalBucket; -import io.github.bucket4j.local.LocalBucketBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.TbCoreConsumerStats; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 09.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService { - - @Value("${transport.remote.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${transport.remote.notifications.topic}") - private String notificationsTopic; - @Value("${transport.remote.rule_engine.poll_interval}") - private int pollDuration; - @Value("${transport.remote.rule_engine.auto_commit_interval}") - private int autoCommitInterval; - - @Value("${transport.remote.rule_engine.poll_records_pack_size}") - private int pollRecordsPackSize; - @Value("${transport.remote.rule_engine.max_poll_records_per_second}") - private long pollRecordsPerSecond; - @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") - private long pollRecordsPerMinute; - @Value("${transport.remote.rule_engine.stats.enabled:false}") - private boolean statsEnabled; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - - @Autowired - private TbCoreQueueProvider coreQueueProvider; - - private TbQueueConsumer> ruleEngineConsumer; - - private TbQueueProducer> notificationsProducer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); - - private volatile boolean stopped = false; - - private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - - @PostConstruct - public void init() { - notificationsProducer = coreQueueProvider.getTransportMsgProducer(); - notificationsProducer.init(); - - //TODO: 2.5 -// ruleEngineConsumer = -// ruleEngineConsumer.subscribe(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - LocalBucketBuilder builder = Bucket4j.builder(); - builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); - builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); - LocalBucket pollRateBucket = builder.build(); - BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = ruleEngineConsumer.poll(pollDuration); - int recordsCount = msgs.size(); - if (recordsCount > 0) { - while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { - log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - log.trace("Processing {} records", recordsCount); - } - msgs.forEach(msg -> { - try { - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - //TODO: 2.5 id - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); - notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - log.trace("Pushing message to local server: {}", toDeviceActorMsg); - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - } - - @PreDestroy - public void destroy() { - stopped = true; - if (ruleEngineConsumer != null) { - ruleEngineConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - private static class QueueCallbackAdaptor implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 4765737430..e84e1b595e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -44,7 +44,8 @@ import java.util.concurrent.*; */ @Slf4j @Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +//TODO 2.5: This Confitional annotation should be removed, and Service renamed to something meaningful +//@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java deleted file mode 100644 index ac1c5965f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; - -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiRequestDecoder implements TbKafkaDecoder { - @Override - public TransportApiRequestMsg decode(byte[] data) throws IOException { - return TransportApiRequestMsg.parseFrom(data); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java deleted file mode 100644 index 35f8a7b1aa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.kafka.TbKafkaEncoder; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiResponseEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(TransportApiResponseMsg value) { - return value.toByteArray(); - } -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index da2e5beb98..57d837de31 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -528,7 +528,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -556,6 +556,7 @@ queue: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java index d717169048..0c34bbf9d6 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,83 +21,92 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TbServiceInfoProvider; +import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) public class ConsistentClusterRoutingServiceTest { - private ConsistentClusterRoutingService clusterRoutingService; + public static final int ITERATIONS = 1000000; + private ConsistentHashPartitionService clusterRoutingService; - private DiscoveryService discoveryService; + private TbServiceInfoProvider discoveryService; private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 1024*4; - private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); + private Integer virtualNodesSize = 16; @Before public void setup() throws Exception { - discoveryService = mock(DiscoveryService.class); - clusterRoutingService = new ConsistentClusterRoutingService(); - ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); + discoveryService = mock(TbServiceInfoProvider.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService); + ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); - List otherServers = new ArrayList<>(); + TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96.1.1") + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build(); +// when(discoveryService.getServiceInfo()).thenReturn(currentServer); + List otherServers = new ArrayList<>(); for (int i = 1; i < 30; i++) { - otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); + otherServers.add(TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96." + i * 2 + "." + i) + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build()); } - when(discoveryService.getOtherServers()).thenReturn(otherServers); clusterRoutingService.init(); + clusterRoutingService.recalculatePartitions(currentServer, otherServers); } @Test public void testDispersionOnMillionDevices() { List devices = new ArrayList<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < ITERATIONS; i++) { devices.add(new DeviceId(UUIDs.timeBased())); } - testDevicesDispersion(devices); } private void testDevicesDispersion(List devices) { long start = System.currentTimeMillis(); - Map map = new HashMap<>(); + Map map = new HashMap<>(); for (DeviceId deviceId : devices) { - ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); - map.put(address, map.getOrDefault(address, 0) + 1); + TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); + Integer partition = address.getPartition().get(); + map.put(partition, map.getOrDefault(partition, 0) + 1); } - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); + double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff/ITERATIONS) * 100.0) + "%)"); - for (Map.Entry entry : data) { -// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); + for (Map.Entry entry : data) { + System.out.println(entry.getKey() + ": " + entry.getValue()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 26d90a26c6..b3778c2f8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.discovery.TopicPartitionInfo; public interface TbQueueProducer { @@ -23,10 +23,6 @@ public interface TbQueueProducer { String getDefaultTopic(); - void send(T msg, TbQueueCallback callback); - - void send(String topic, T msg, TbQueueCallback callback); - - ListenableFuture send(String topic, int partition, T msg, TbQueueCallback callback); + void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index d022dafadc..53914953f3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -28,6 +28,7 @@ import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -160,7 +161,7 @@ public class DefaultTbQueueRequestTemplate responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); - requestTemplate.send(request, new TbQueueCallback() { + requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { log.trace("[{}] Request sent: {}", requestId, metadata); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 3804fb5160..c77692ee36 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueHandler; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -108,7 +109,7 @@ public class DefaultTbQueueResponseTemplate { pendingRequestCount.decrementAndGet(); response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(responseTopic, response, null); + responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); }, e -> { pendingRequestCount.decrementAndGet(); diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java index 711677dc78..f96fbd4ece 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.discovery; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -25,11 +24,10 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = - new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); - public void put(long hash, ServerInstance instance) { + public void put(long hash, T instance) { circle.put(hash, instance); } @@ -45,7 +43,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -53,11 +51,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public ServerInstance get(Long hash) { + public T get(Long hash) { return circle.get(hash); } public void log() { - circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); + circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java new file mode 100644 index 0000000000..c9b7b04e70 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java @@ -0,0 +1,221 @@ +package org.thingsboard.server.discovery; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; + +@Service +@Slf4j +public class ConsistentHashPartitionService implements PartitionService { + + @Value("${queue.core.topic}") + private String coreTopic; + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${queue.rule_engine.partitions:100}") + private Integer ruleEnginePartitions; + @Value("${queue.partitions.hash_function_name:murmur3_32}") + private String hashFunctionName; + @Value("${queue.partitions.virtual_nodes_size:16}") + private Integer virtualNodesSize; + + private final TbServiceInfoProvider serviceInfoProvider; + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. + private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + + private HashFunction hashFunction; + + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider) { + this.serviceInfoProvider = serviceInfoProvider; + } + + @PostConstruct + public void init() { + this.hashFunction = forName(hashFunctionName); + partitionSizes.put(ServiceType.TB_CORE, corePartitions); + partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); + partitionTopics.put(ServiceType.TB_CORE, coreTopic); + partitionTopics.put(ServiceType.TB_RULE_ENGINE, ruleEngineTopic); + } + + @Override + public List getCurrentPartitions(ServiceType serviceType) { + ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + TenantId tenantId = getTenantId(currentService); + ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); + List partitions = myPartitions.get(serviceKey); + List topicPartitions = new ArrayList<>(); + for (Integer partition : partitions) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (!tenantId.isNullUid()) { + tpi.tenantId(tenantId); + } + topicPartitions.add(tpi.build()); + } + return topicPartitions; + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + int hash = hashFunction.newHasher() + .putLong(entityId.getId().getMostSignificantBits()) + .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); + int partition = Math.abs(hash % partitionSizes.get(serviceType)); + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (isolated) { + tpi.tenantId(tenantId); + } + return tpi.build(); + } + + @Override + public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + logServiceInfo(currentService); + otherServices.forEach(this::logServiceInfo); + + Map> newCircles = new HashMap<>(ServiceType.values().length); + for (ServiceType serverType : ServiceType.values()) { + newCircles.put(serverType, new ConsistentHashCircle<>()); + } + addNode(newCircles, currentService); + for (ServiceInfo other : otherServices) { + addNode(newCircles, other); + TenantId tenantId = getTenantId(other); + if (!tenantId.isNullUid()) { + isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); + for (String serviceType : other.getServiceTypesList()) { + isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); + } + + } + } + ConcurrentMap> oldPartitions = myPartitions; + myPartitions = new ConcurrentHashMap<>(); + partitionSizes.forEach((type, size) -> { + for (int i = 0; i < size; i++) { + ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); + if (currentService.equals(serviceInfo)) { + myPartitions.putIfAbsent(new ServiceKey(type, getTenantId(serviceInfo)), new ArrayList<>()); + } + } + }); + myPartitions.forEach((serviceKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + } + }); + } + + private void logServiceInfo(TransportProtos.ServiceInfo server) { + TenantId tenantId = getTenantId(server); + if (tenantId.isNullUid()) { + log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); + } else { + log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); + } + } + + private TenantId getTenantId(TransportProtos.ServiceInfo serviceInfo) { + return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); + } + + private void addNode(Map> circles, ServiceInfo instance) { + for (String serviceTypeStr : instance.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.get(serviceType).put(hash(instance, i).asLong(), instance); + } + } + } + + private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { + if (circle.isEmpty()) { + return null; + } + Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + return circle.get(hash); + } + + private HashCode hash(ServiceInfo instance, int i) { + return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); + } + + private static class ServiceKey { + @Getter + private final ServiceType serviceType; + @Getter + private final TenantId tenantId; + + public ServiceKey(ServiceType serviceType, TenantId tenantId) { + this.serviceType = serviceType; + this.tenantId = tenantId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceKey that = (ServiceKey) o; + return serviceType == that.serviceType && + Objects.equals(tenantId, that.tenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, tenantId); + } + } + + public static HashFunction forName(String name) { + switch (name) { + case "murmur3_32": + return Hashing.murmur3_32(); + case "murmur3_128": + return Hashing.murmur3_128(); + case "crc32": + return Hashing.crc32(); + case "md5": + return Hashing.md5(); + default: + throw new IllegalArgumentException("Can't find hash function with name " + name); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java index 0b09a333d8..1fd87a40b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -69,11 +69,14 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { ServiceInfo.Builder builder = ServiceInfo.newBuilder() .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + UUID tenantId; if (!StringUtils.isEmpty(tenantIdStr)) { - UUID tenantId = UUID.fromString(tenantIdStr); - builder.setTenantIdMSB(tenantId.getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + tenantId = UUID.fromString(tenantIdStr); + } else { + tenantId = TenantId.NULL_UUID; } + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java new file mode 100644 index 0000000000..ef9aca72a0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -0,0 +1,32 @@ +package org.thingsboard.server.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import java.util.Collections; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@Slf4j +@DependsOn("environmentLogService") +public class DummyDiscoveryService implements PartitionDiscoveryService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + + + public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java index d85b1d3f44..6fe4571517 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java @@ -15,15 +15,6 @@ */ package org.thingsboard.server.discovery; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.List; - public interface PartitionDiscoveryService { - List getCurrentPartitions(ServiceType serviceType); - - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); - } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java new file mode 100644 index 0000000000..90d43f4af3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java @@ -0,0 +1,16 @@ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; + +public interface PartitionService { + + List getCurrentPartitions(ServiceType serviceType); + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + + void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java index b46b941360..4480d07dd6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java @@ -15,12 +15,27 @@ */ package org.thingsboard.server.discovery; -import lombok.Data; +import lombok.Builder; +import org.thingsboard.server.common.data.id.TenantId; -@Data +import java.util.Optional; + +@Builder public class TopicPartitionInfo { - private String topic; - private int partition; + private final String topic; + private final TenantId tenantId; + private final Integer partition; + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + public Optional getPartition() { + return Optional.ofNullable(partition); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java index a722f2f395..d9af5bb894 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,11 +15,13 @@ */ package org.thingsboard.server.discovery; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -39,16 +41,21 @@ import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @@ -66,16 +73,8 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Value("${zk.zk_dir}") private String zkDir; - @Value("${queue.core.partitions:100}") - private Integer corePartitions; - @Value("${queue.rule_engine.partitions:100}") - private Integer ruleEnginePartitions; - - @Autowired - private TbServiceInfoProvider serviceIdProvider; - - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private final ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private ExecutorService reconnectExecutorService; private CuratorFramework client; @@ -85,14 +84,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P private volatile boolean stopped = true; - @Override - public List getCurrentPartitions(ServiceType serviceType) { - return Collections.emptyList(); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - + public ZkPartitionDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; } @PostConstruct @@ -105,15 +99,26 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - partitionSizes.put(ServiceType.TB_CORE, corePartitions); - partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); zkNodesDir = zkDir + "/nodes"; initZkClient(); } + private List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return TransportProtos.ServiceInfo.parseFrom(cd.getData()); + } catch (NoSuchElementException | InvalidProtocolBufferException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { if (stopped) { @@ -127,23 +132,20 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return; } publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); } - @Override public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); } else { try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + log.info("[{}] Creating ZK node for current instance", self.getServiceId()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); + log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -157,10 +159,10 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + TransportProtos.ServiceInfo registeredServerInfo = null; + registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); + if (self.equals(registeredServerInfo)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -171,9 +173,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } - private ConnectionStateListener checkReconnect(ServerInstance self) { + private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -250,6 +252,46 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - + if (stopped) { + log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); + return; + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); + return; + } + ChildData data = pathChildrenCacheEvent.getData(); + if (data == null) { + log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); + return; + } else if (data.getData() == null) { + log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); + return; + } else if (nodePath != null && nodePath.equals(data.getPath())) { + if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { + log.info("ZK node for current instance is somehow deleted."); + publishCurrentServer(); + } + log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); + return; + } + TransportProtos.ServiceInfo instance; + try { + instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode server instance for node {}", data.getPath(), e); + throw e; + } + log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + switch (pathChildrenCacheEvent.getType()) { + case CHILD_ADDED: + case CHILD_UPDATED: + case CHILD_REMOVED: + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + break; + default: + break; + } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java index a6a881ecde..cff3fd1056 100644 --- a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.environment; +package org.thingsboard.server.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("Thingsboard server environment: ", log); + Environment.logEnv("ThingsBoard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index d76a2dc625..8072a2da66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.Properties; @@ -46,10 +47,6 @@ public class TBKafkaProducerTemplate implements TbQueuePro private final KafkaProducer producer; - private final TbKafkaPartitioner partitioner; - - private ConcurrentMap> partitionInfoMap; - @Getter private final String defaultTopic; @@ -66,41 +63,28 @@ public class TBKafkaProducerTemplate implements TbQueuePro } this.settings = settings; this.producer = new KafkaProducer<>(props); - this.partitioner = partitioner; this.defaultTopic = defaultTopic; } - public void init() { - this.partitionInfoMap = new ConcurrentHashMap<>(); - if (!StringUtils.isEmpty(defaultTopic)) { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); - admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); - log.info("[{}] Topic exists.", defaultTopic); - } catch (Exception e) { - log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); - throw new RuntimeException(e); - } - //Maybe this should not be cached, but we don't plan to change size of partitions - this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); - } - } - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); + public void init() { } @Override - public void send(String topic, T msg, TbQueueCallback callback) { + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - - Integer partition = getPartition(topic, msg); - record = new ProducerRecord<>(topic, partition, key, data, headers); - Future result = producer.send(record, (metadata, exception) -> { + StringBuilder topic = new StringBuilder().append(tpi.getTopic()); + if (tpi.getTenantId().isPresent()) { + topic.append(".").append(tpi.getTenantId().get().getId().toString()); + } + if (tpi.getPartition().isPresent()) { + topic.append(".").append(tpi.getPartition().get()); + } + record = new ProducerRecord<>(topic.toString(), null, key, data, headers); + producer.send(record, (metadata, exception) -> { if (exception == null) { callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); } else { @@ -108,12 +92,4 @@ public class TBKafkaProducerTemplate implements TbQueuePro } }); } - - private Integer getPartition(String topic, T value) { - if (partitioner == null) { - return null; - } else { - return partitioner.partition(topic, value.getKey().toString(), value, value.getData(), partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index e71d579c7f..8abb5e2d43 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { @@ -37,18 +38,8 @@ public class InMemoryTbQueueProducer implements TbQueuePro } @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); - } - - @Override - public void send(String topic, T msg, TbQueueCallback callback) { - boolean result = storage.put(topic, msg); + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + boolean result = storage.put(tpi.getTopic(), msg); if (result) { callback.onSuccess(null); } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index 79f5dd7da0..c7839e70e9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -32,12 +32,12 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; @Slf4j @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='in-memory'") -public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbQueueCoreSettings coreSettings; - public InMemoryTbCoreQueueProvider(TbQueueCoreSettings coreSettings) { + public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings) { this.coreSettings = coreSettings; } @@ -59,6 +59,12 @@ public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { return producer; } + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + @Override public TbQueueConsumer> getToCoreMsgConsumer() { InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java new file mode 100644 index 0000000000..6aaf76420b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index d830cff8fa..426a30eea1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +32,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbKafkaSettings kafkaSettings; @@ -49,7 +49,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getTransportMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,7 +58,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -74,28 +74,33 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { @Override public TbQueueConsumer> getToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(coreSettings.getTopic()); responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } - @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { - return null; - } - @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..6057ebba6f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 3b671a3adb..c7b5b6f839 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -35,7 +35,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @@ -62,9 +62,7 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..46f1a3351d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbRuleEngineQueueProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToRuleEngineMsgConsumer(); + +} diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index d6dcbbabc6..af471e537c 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,6 +36,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common data diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index cdf6b3f763..661411cf89 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,6 +34,9 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.discovery.PartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -76,8 +79,8 @@ public class DefaultTransportService implements TransportService { @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; - @Autowired - private TransportQueueProvider queueProvider; + private final TransportQueueProvider queueProvider; + private final PartitionService partitionService; @Value("${kafka.notifications.poll_interval}") private int notificationsPollDuration; @@ -98,6 +101,11 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; + public DefaultTransportService(TransportQueueProvider queueProvider, PartitionService partitionService) { + this.queueProvider = queueProvider; + this.partitionService = partitionService; + } + @PostConstruct public void init() { if (rateLimitEnabled) { @@ -420,6 +428,14 @@ public class DefaultTransportService implements TransportService { return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } + protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { + return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + } + + protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { + return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + } + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { return TransportProtos.SessionEventMsg.newBuilder() .setSessionType(TransportProtos.SessionType.ASYNC) @@ -427,15 +443,19 @@ public class DefaultTransportService implements TransportService { } protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + tbCoreMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + ruleEngineMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } private class TransportTbQueueCallback implements TbQueueCallback { From 2a815058e7279ee4aefbcaa2c5ba3adff00b7532 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 12:38:28 +0200 Subject: [PATCH 013/102] Improvements and bugfix to QueueProviders --- .../server/actors/ActorSystemContext.java | 23 ++++---- .../actors/ruleChain/DefaultTbContext.java | 4 +- .../RuleChainActorMessageProcessor.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 11 +++- .../DefaultTbRuleEngineConsumerService.java | 16 ++++-- .../service/queue/TbCoreConsumerService.java | 4 +- .../queue/TbRuleEngineConsumerService.java | 5 +- .../queue/TbRuleEngineConsumerStats.java | 2 +- .../DefaultTbCoreToTransportService.java | 4 +- .../src/main/resources/thingsboard.yml | 21 +++----- ...=> ConsistentHashParitionServiceTest.java} | 13 ++--- .../thingsboard/server/TbQueueConsumer.java | 2 + .../thingsboard/server/TbQueueProducer.java | 2 +- .../server/TbQueueRequestTemplate.java | 4 ++ .../server/TbQueueRuleEngineSettings.java | 31 +++++++++++ .../server/TbQueueTransportApiSettings.java | 7 +++ .../TbQueueTransportNotificationSettings.java | 32 +++++++++++ .../common/DefaultTbQueueRequestTemplate.java | 4 +- .../discovery/ConsistentHashCircle.java | 2 +- .../ConsistentHashPartitionService.java | 54 ++++++++----------- .../DefaultTbServiceInfoProvider.java | 2 +- .../discovery/DummyDiscoveryService.java | 15 ++++++ .../discovery/PartitionChangeEvent.java | 6 ++- .../server/discovery/PartitionService.java | 15 ++++++ .../server/discovery/ServiceKey.java | 47 ++++++++++++++++ .../ZkPartitionDiscoveryService.java | 2 +- .../server/kafka/TBKafkaConsumerTemplate.java | 40 +++++++++----- .../server/kafka/TBKafkaProducerTemplate.java | 2 +- .../server/memory/InMemoryStorage.java | 39 +++++++++++--- .../memory/InMemoryTbQueueConsumer.java | 8 ++- .../memory/InMemoryTbQueueProducer.java | 11 ++-- .../InMemoryMonolithQueueProvider.java | 36 +++++++------ .../InMemoryTransportQueueProvider.java | 28 +++++++--- .../provider/KafkaMonolithQueueProvider.java | 4 +- .../provider/KafkaTbCoreQueueProvider.java | 4 +- .../KafkaTbRuleEngineQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 2 +- .../server/provider/TbCoreQueueProvider.java | 2 + .../provider/TbRuleEngineQueueProvider.java | 2 +- .../service/DefaultTransportService.java | 12 +++-- 40 files changed, 384 insertions(+), 138 deletions(-) rename application/src/test/java/org/thingsboard/server/service/cluster/routing/{ConsistentClusterRoutingServiceTest.java => ConsistentHashParitionServiceTest.java} (92%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e5a0e6a25a..3db5c82ee2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -35,6 +35,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; @@ -64,6 +65,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.discovery.TbServiceInfoProvider; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; @@ -102,6 +104,11 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired + @Getter + @Setter + private TbServiceInfoProvider serviceInfoProvider; + @Getter @Setter private ActorService actorService; @@ -233,10 +240,6 @@ public class ActorSystemContext { @Getter private RuleChainTransactionService ruleChainTransactionService; - @Value("${cluster.partition_id}") - @Getter - private long queuePartitionId; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -359,7 +362,7 @@ public class ActorSystemContext { event.setEntityId(entityId); event.setType(DataConstants.ERROR); //TODO 2.5 -// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -369,7 +372,7 @@ public class ActorSystemContext { event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); //TODO 2.5 -// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -383,8 +386,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); + private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -394,8 +397,8 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(ServerAddress server, String method, String body) { - return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); + private JsonNode toBodyJson(String serviceId, String method, String body) { + return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index f5ea5c399f..3e982b520e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -146,12 +146,12 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), 0); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), 0); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 095af7f62d..fe95c448a0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -331,6 +331,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ef2b44bb6c..6f8d9dc9ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import akka.actor.ActorRef; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -27,10 +26,10 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbCoreQueueProvider; import org.thingsboard.server.provider.TbRuleEngineQueueProvider; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -70,10 +69,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PostConstruct public void init() { - this.consumer.subscribe(); this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); } + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.consumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { mainConsumerExecutor.execute(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 1e1b352645..8a39b5df4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.queue; +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.function.Consumer; -public interface TbCoreConsumerService { +public interface TbCoreConsumerService extends ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 671cd72262..968fbb2274 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.service.queue; -public interface TbRuleEngineConsumerService { +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.discovery.PartitionChangeEvent; + +public interface TbRuleEngineConsumerService extends ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index a5e8ba94af..27792c5a61 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index f7afd105dc..a6e1a37eca 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,7 +40,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService private final TbQueueProducer> tbTransportProducer; - @Value("${queue.notifications.topic}") + @Value("${queue.transport.notifications_topic}") private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 57d837de31..fc49716d25 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -70,16 +70,7 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" - # Name of hash function used for consistent hash ring. - hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" - # Amount of virtual nodes in consistent hash ring. - vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" - # Queue partition id for current node - partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -536,6 +527,9 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" @@ -547,7 +541,7 @@ queue: core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" @@ -555,13 +549,14 @@ queue: rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + transport: + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java similarity index 92% rename from application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 0c34bbf9d6..9442202688 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,11 +21,10 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; import org.thingsboard.server.discovery.ConsistentHashPartitionService; import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.discovery.TbServiceInfoProvider; @@ -44,12 +43,13 @@ import static org.mockito.Mockito.mock; @Slf4j @RunWith(MockitoJUnitRunner.class) -public class ConsistentClusterRoutingServiceTest { +public class ConsistentHashParitionServiceTest { public static final int ITERATIONS = 1000000; private ConsistentHashPartitionService clusterRoutingService; private TbServiceInfoProvider discoveryService; + private ApplicationEventPublisher applicationEventPublisher; private String hashFunctionName = "murmur3_128"; private Integer virtualNodesSize = 16; @@ -58,7 +58,8 @@ public class ConsistentClusterRoutingServiceTest { @Before public void setup() throws Exception { discoveryService = mock(TbServiceInfoProvider.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService); + applicationEventPublisher = mock(ApplicationEventPublisher.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, applicationEventPublisher); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); @@ -103,7 +104,7 @@ public class ConsistentClusterRoutingServiceTest { List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff/ITERATIONS) * 100.0) + "%)"); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); for (Map.Entry entry : data) { System.out.println(entry.getKey() + ": " + entry.getValue()); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ddf9d7d9b3..ac4d7611f6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -23,6 +23,8 @@ public interface TbQueueConsumer { void subscribe(); + void subscribe(List partitions); + void unsubscribe(); List poll(long durationInMillis); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index b3778c2f8e..2498117c46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5182bb7260..2eb429b082 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -19,6 +19,10 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { + void init(); + ListenableFuture send(Request request); + void stop(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..4acfec1091 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRuleEngineSettings { + + @Value("${queue.rule_engine.topic}") + private String topic; + + @Value("${queue.rule_engine.partitions}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java index 603d787fa8..b98bd3b76d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; @Data @Component public class TbQueueTransportApiSettings { + @Value("${queue.transport_api.requests_topic}") private String requestsTopic; @@ -34,6 +35,12 @@ public class TbQueueTransportApiSettings { @Value("${queue.transport_api.max_requests_timeout}") private int maxRequestsTimeout; + @Value("${queue.transport_api.max_callback_threads}") + private int maxCallbackThreads; + + @Value("${queue.transport_api.request_poll_interval}") + private long requestPollInterval; + @Value("${queue.transport_api.response_poll_interval}") private long responsePollInterval; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java new file mode 100644 index 0000000000..2666045e7e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportNotificationSettings { + + @Value("${queue.transport.notifications_topic}") + private String notificationsTopic; + + @Value("${queue.transport.poll_interval}") + private long transportPollInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 53914953f3..0388daebb6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -79,6 +79,7 @@ public class DefaultTbQueueRequestTemplate partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); @@ -53,8 +68,9 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { this.serviceInfoProvider = serviceInfoProvider; + this.applicationEventPublisher = applicationEventPublisher; } @PostConstruct @@ -128,13 +144,15 @@ public class ConsistentHashPartitionService implements PartitionService { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); if (currentService.equals(serviceInfo)) { - myPartitions.putIfAbsent(new ServiceKey(type, getTenantId(serviceInfo)), new ArrayList<>()); + ServiceKey serviceKey = new ServiceKey(type, getTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); } } }); myPartitions.forEach((serviceKey, partitions) -> { if (!partitions.equals(oldPartitions.get(serviceKey))) { log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, partitions)); } }); } @@ -178,32 +196,6 @@ public class ConsistentHashPartitionService implements PartitionService { return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); } - private static class ServiceKey { - @Getter - private final ServiceType serviceType; - @Getter - private final TenantId tenantId; - - public ServiceKey(ServiceType serviceType, TenantId tenantId) { - this.serviceType = serviceType; - this.tenantId = tenantId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceKey that = (ServiceKey) o; - return serviceType == that.serviceType && - Objects.equals(tenantId, that.tenantId); - } - - @Override - public int hashCode() { - return Objects.hash(serviceType, tenantId); - } - } - public static HashFunction forName(String name) { switch (name) { case "murmur3_32": diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java index 1fd87a40b6..80040a017b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java index ef9aca72a0..8983cf1e97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.discovery; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java index 7d083ba873..570a774c9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java @@ -16,6 +16,7 @@ package org.thingsboard.server.discovery; import lombok.Getter; +import lombok.Setter; import org.springframework.context.ApplicationEvent; import java.util.List; @@ -23,11 +24,14 @@ import java.util.List; public class PartitionChangeEvent extends ApplicationEvent { + @Getter + private final ServiceKey serviceKey; @Getter private final List partitions; - public PartitionChangeEvent(Object source, List partitions) { + public PartitionChangeEvent(Object source, ServiceKey serviceKey, List partitions) { super(source); + this.serviceKey = serviceKey; this.partitions = partitions; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java index 90d43f4af3..ad61a63807 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.discovery; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java new file mode 100644 index 0000000000..0179be968c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +public class ServiceKey { + @Getter + private final ServiceType serviceType; + @Getter + private final TenantId tenantId; + + public ServiceKey(ServiceType serviceType, TenantId tenantId) { + this.serviceType = serviceType; + this.tenantId = tenantId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceKey that = (ServiceKey) o; + return serviceType == that.serviceType && + Objects.equals(tenantId, that.tenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, tenantId); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java index d9af5bb894..07f11617ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 558d556d09..51b8725118 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -40,6 +41,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; + private volatile boolean subscribed; @Getter private final String topic; @@ -69,23 +71,37 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { consumer.subscribe(Collections.singletonList(topic)); + subscribed = true; + } + + @Override + public void subscribe(List partitions) { + consumer.subscribe(partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList())); + subscribed = true; } @Override public List poll(long durationInMillis) { - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; + if (!subscribed) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } } - return Collections.emptyList(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 8072a2da66..1b5d57f1d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index ded4cd9810..86d103ea70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -15,16 +15,24 @@ */ package org.thingsboard.server.memory; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.TbQueueMsg; +import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +@Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; - private final Map> storage; + private final Map> storage; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); @@ -42,19 +50,38 @@ public final class InMemoryStorage { } public boolean put(String topic, TbQueueMsg msg) { - return storage.computeIfAbsent(topic, (t) -> new LinkedList<>()).add(msg); + return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public TbQueueMsg get(String topic) { + public List get(String topic, long durationInMillis) { if (storage.containsKey(topic)) { - return storage.get(topic).peek(); + try { + List entities; + T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + } else { + entities = Collections.emptyList(); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 100); + for (TbQueueMsg other : otherList) { + entities.add((T) other); + } + } + return entities; + } catch (InterruptedException e) { + log.warn("Queue was interrupted", e); + return Collections.emptyList(); + } } - return null; + return Collections.emptyList(); } public void commit(String topic) { + //TODO: 2.5 Until someone calls commit you should not allow to poll new elements. if (storage.containsKey(topic)) { - storage.get(topic).remove(); +// storage.get(topic).remove(); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index b8ddcff89c..3f1e844efc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -18,7 +18,6 @@ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueMsg; -import java.util.Collections; import java.util.List; public class InMemoryTbQueueConsumer implements TbQueueConsumer { @@ -40,6 +39,11 @@ public class InMemoryTbQueueConsumer implements TbQueueCon } + @Override + public void subscribe(List partitions) { + + } + @Override public void unsubscribe() { @@ -47,7 +51,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public List poll(long durationInMillis) { - return Collections.singletonList((T)storage.get(topic)); + return storage.get(topic, durationInMillis); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 8abb5e2d43..79fa1d6b39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -41,10 +41,13 @@ public class InMemoryTbQueueProducer implements TbQueuePro public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { boolean result = storage.put(tpi.getTopic(), msg); if (result) { - callback.onSuccess(null); + if (callback != null) { + callback.onSuccess(null); + } } else { - Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); - callback.onFailure(e); + if (callback != null) { + callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); + } } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index c7839e70e9..4fa10b7c44 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -21,7 +21,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRuleEngineSettings; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -36,50 +40,52 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings) { + public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.notificationSettings = notificationSettings; } @Override public TbQueueProducer> getTransportMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> getRuleEngineMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueConsumer> getToCoreMsgConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> getTransportApiResponseProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java index 789ea1cf5a..2d7205e516 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java @@ -15,13 +15,19 @@ */ package org.thingsboard.server.provider; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueAdmin; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueRuleEngineSettings; import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,20 +43,29 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; @Slf4j public class InMemoryTransportQueueProvider implements TransportQueueProvider { + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings) { + public InMemoryTransportQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.notificationSettings = notificationSettings; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(topic -> Futures.immediateFuture(null)); templateBuilder.requestTemplate(producer); templateBuilder.responseTemplate(consumer); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -61,19 +76,16 @@ public class InMemoryTransportQueueProvider implements TransportQueueProvider { @Override public TbQueueProducer> getRuleEngineMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - return producer; + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - return producer; + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getTransportNotificationsConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index 6aaf76420b..f06897ebe6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,8 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 426a30eea1..5cbe93196a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,8 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java index 6057ebba6f..eb0bcb5d6e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index c7b5b6f839..dcf6474e1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index 6123fc0859..ef9cd23ac3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.provider; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java index 46f1a3351d..2efa9ffbc6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 661411cf89..19bf43f7db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -78,13 +78,12 @@ public class DefaultTransportService implements TransportService { private long sessionInactivityTimeout; @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Value("${queue.transport.poll_interval}") + private int notificationsPollDuration; private final TransportQueueProvider queueProvider; private final PartitionService partitionService; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; protected TbQueueProducer> tbCoreMsgProducer; @@ -120,7 +119,7 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); - + transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -163,6 +162,9 @@ public class DefaultTransportService implements TransportService { if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (transportApiRequestTemplate != null) { + transportApiRequestTemplate.stop(); + } } @Override From ea4f35637660d82cc925a4f1cb46541477cb205f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 12:51:03 +0200 Subject: [PATCH 014/102] Consumer improvements --- .../server/service/queue/DefaultTbCoreConsumerService.java | 3 +++ .../service/queue/DefaultTbRuleEngineConsumerService.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 0907b8be60..1d3cf11f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -88,6 +88,9 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { while (!stopped) { try { List> msgs = consumer.poll(pollDuration); + if(msgs.isEmpty()){ + continue; + } ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 6f8d9dc9ac..253c7d9933 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -86,6 +86,9 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS while (!stopped) { try { List> msgs = consumer.poll(pollDuration); + if(msgs.isEmpty()){ + continue; + } ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); @@ -120,7 +123,8 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } //TODO 2.5 - private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toDeviceActorMsg, TbMsgCallback callback) { + private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + log.info("Received RULE ENGINE msg: {}", toRuleEngineMsg); // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } From 7de485f4535c86e6cbb3ddafb44d1784898f8d4f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 14:26:44 +0200 Subject: [PATCH 015/102] Kafka configuration --- .../DefaultTbCoreToTransportService.java | 2 +- .../org/thingsboard/server/TbQueueAdmin.java | 4 +- .../server/kafka/TBKafkaAdmin.java | 37 ++++++++--- .../server/kafka/TBKafkaConsumerTemplate.java | 11 +++- .../server/kafka/TBKafkaProducerTemplate.java | 10 ++- .../server/kafka/TbKafkaSettings.java | 5 +- .../InMemoryMonolithQueueProvider.java | 2 +- .../provider/KafkaMonolithQueueProvider.java | 63 +++++++++++-------- .../provider/KafkaTbCoreQueueProvider.java | 5 +- .../KafkaTbRuleEngineQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 39 +++++++++--- .../server/provider/TbCoreQueueProvider.java | 2 +- .../provider/TbRuleEngineQueueProvider.java | 2 +- .../service/DefaultTransportService.java | 1 + 14 files changed, 122 insertions(+), 63 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index a6e1a37eca..833b28c3b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -44,7 +44,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); + this.tbTransportProducer = tbCoreQueueProvider.getTransportNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 6917c23719..5952e7da70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -15,10 +15,8 @@ */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; - public interface TbQueueAdmin { - ListenableFuture createTopicIfNotExists(String topic); + void createTopicIfNotExists(String topic); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 0bb3a75db6..5f1b41ea7b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -18,11 +18,13 @@ package org.thingsboard.server.kafka; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.JdkFutureAdapters; import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.TbQueueAdmin; import java.util.Collections; @@ -33,6 +35,7 @@ import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. */ +@Slf4j public class TBKafkaAdmin implements TbQueueAdmin { AdminClient client; @@ -41,17 +44,31 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } + //TODO 2.5 @Override - public ListenableFuture createTopicIfNotExists(String topic) { - - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - - ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); - - return Futures.transformAsync(topicFuture, topicDescription -> { - KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); - return JdkFutureAdapters.listenInPoolThread(resultFuture); - }); + public void createTopicIfNotExists(String topic) { + try { + createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic).get(); + } catch (ExecutionException ee) { + if (ee.getCause() instanceof TopicExistsException) { + //do nothing + } else { + log.warn("[{}] Failed to create topic", topic, ee); + throw new RuntimeException(ee); + } + } catch (Exception e) { + log.warn("[{}] Failed to create topic", topic, e); + throw new RuntimeException(e); + } +// +// KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); +// +// ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); +// +// return Futures.transformAsync(topicFuture, topicDescription -> { +// KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); +// return JdkFutureAdapters.listenInPoolThread(resultFuture); +// }); } public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 51b8725118..eb35c583c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; /** @@ -39,6 +40,7 @@ import java.util.stream.Collectors; @Slf4j public class TBKafkaConsumerTemplate implements TbQueueConsumer { + private final TBKafkaAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; @@ -63,6 +65,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon if (maxPollRecords > 0) { props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); } + this.admin = new TBKafkaAdmin(settings); this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; @@ -70,13 +73,16 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { + createTopicIfNotExists(topic); consumer.subscribe(Collections.singletonList(topic)); subscribed = true; } @Override public void subscribe(List partitions) { - consumer.subscribe(partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList())); + List topicNames = partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList()); + topicNames.forEach(this::createTopicIfNotExists); + consumer.subscribe(topicNames); subscribed = true; } @@ -119,4 +125,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon return decoder.decode(new KafkaTbQueueMsg(record)); } + private void createTopicIfNotExists(String topic) { + admin.createTopicIfNotExists(topic); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 1b5d57f1d2..4999ba026a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -86,9 +86,15 @@ public class TBKafkaProducerTemplate implements TbQueuePro record = new ProducerRecord<>(topic.toString(), null, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { - callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + if (callback != null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } } else { - callback.onFailure(exception); + if (callback != null) { + callback.onFailure(exception); + } else { + log.warn("Producer template failure", exception); + } } }); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 566766fee6..912e935bef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -18,6 +18,7 @@ package org.thingsboard.server.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -28,7 +29,7 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") @Component public class TbKafkaSettings { @@ -41,7 +42,7 @@ public class TbKafkaSettings { @Value("${queue.kafka.acks}") private String acks; - @Value("${queue.queue.kafka.retries}") + @Value("${queue.kafka.retries}") private int retries; @Value("${queue.kafka.batch.size}") diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index 4fa10b7c44..7fc7bdea7c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -55,7 +55,7 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index f06897ebe6..01e7a211d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -20,9 +20,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRuleEngineSettings; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -40,19 +41,30 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbKafkaSettings kafkaSettings; private final TbNodeIdProvider nodeIdProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, + TbNodeIdProvider nodeIdProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.nodeIdProvider = nodeIdProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.clientId("producer-transport-notifications" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); return requestBuilder.build(); } @@ -61,7 +73,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -76,14 +88,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override @@ -92,30 +103,28 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-consumer-" + nodeIdProvider.getNodeId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - //TODO 2.5 - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("transport-api-consumer-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.clientId("transport-api-producer-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 5cbe93196a..72338bb149 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -21,8 +21,6 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -48,7 +46,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); @@ -93,7 +91,6 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO 2.5 responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java index eb0bcb5d6e..d635419141 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -46,7 +46,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index dcf6474e1e..577c9180fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -19,9 +19,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueRuleEngineSettings; import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -29,6 +32,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaAdmin; import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.kafka.TbKafkaSettings; @@ -41,31 +45,42 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbKafkaSettings kafkaSettings; private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueTransportApiSettings transportApiSettings) { + public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, + TbNodeIdProvider nodeIdProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-transport-api-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("transport-node-" + nodeIdProvider.getNodeId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); templateBuilder.requestTemplate(requestBuilder.build()); templateBuilder.responseTemplate(responseBuilder.build()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -78,7 +93,7 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); return requestBuilder.build(); } @@ -87,13 +102,19 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-tb-core-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); return requestBuilder.build(); } @Override public TbQueueConsumer> getTransportNotificationsConsumer() { - return null; + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index ef9cd23ac3..ba93bf96ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -37,7 +37,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getTransportMsgProducer(); + TbQueueProducer> getTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java index 2efa9ffbc6..b53c3ac28b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -35,7 +35,7 @@ public interface TbRuleEngineQueueProvider { * * @return */ - TbQueueProducer> getTransportMsgProducer(); + TbQueueProducer> getTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 19bf43f7db..f1b73f30af 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -119,6 +119,7 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { From 94eb213716f0cb19364bb08d938c9f980872d2db Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 18:38:09 +0200 Subject: [PATCH 016/102] Implementation --- .../server/actors/ActorSystemContext.java | 6 +- .../device/DeviceActorMessageProcessor.java | 3 + .../server/actors/stats/StatsActor.java | 9 +- .../queue/DefaultTbCoreConsumerService.java | 24 +- .../server/service/queue/MsgPackCallback.java | 2 + .../service/queue/TbCoreConsumerStats.java | 5 + .../state/DefaultDeviceStateService.java | 299 +++++++++--------- .../service/state/DeviceStateService.java | 10 +- application/src/main/proto/cluster.proto | 10 - application/src/main/resources/logback.xml | 2 + .../common/AbstractTbQueueTemplate.java | 9 +- .../ConsistentHashPartitionService.java | 1 + ...veryService.java => DiscoveryService.java} | 2 +- .../discovery/DummyDiscoveryService.java | 2 +- ...ryService.java => ZkDiscoveryService.java} | 12 +- .../provider/KafkaMonolithQueueProvider.java | 22 +- .../proto/{transport.proto => queue.proto} | 15 + 17 files changed, 222 insertions(+), 211 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/discovery/{PartitionDiscoveryService.java => DiscoveryService.java} (93%) rename common/queue/src/main/java/org/thingsboard/server/discovery/{ZkPartitionDiscoveryService.java => ZkDiscoveryService.java} (95%) rename common/queue/src/main/proto/{transport.proto => queue.proto} (95%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 3db5c82ee2..6b8e950fdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -361,7 +361,6 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - //TODO 2.5 event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -371,7 +370,6 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - //TODO 2.5 event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -403,9 +401,7 @@ public class ActorSystemContext { public String getServerAddress() { - //TODO 2.5 -// return discoveryService.getCurrentServer().getServerAddress().toString(); - return null; + return serviceInfoProvider.getServiceId(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6b878ef7d8..6b5dfbf6c3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -65,6 +65,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; @@ -225,6 +226,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); + TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -258,6 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (reportDeviceActivity) { reportLogicalDeviceActivity(); } + callback.onSuccess(); } //TODO 2.5 move this as a notification to the queue; diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index c8e55bf9ed..8757deb2b4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -58,13 +58,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - //TODO 2.5 -// event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 1d3cf11f40..c95599f2ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,9 +29,11 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -59,14 +61,16 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private boolean statsEnabled; private final ActorSystemContext actorContext; + private final DeviceStateService stateService; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService) { this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); this.actorContext = actorContext; + this.stateService = stateService; } @PostConstruct @@ -88,7 +92,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { while (!stopped) { try { List> msgs = consumer.poll(pollDuration); - if(msgs.isEmpty()){ + if (msgs.isEmpty()) { continue; } ConcurrentMap> ackMap = msgs.stream().collect( @@ -98,11 +102,12 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { ToCoreMsg toCoreMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toCoreMsg); if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); - } else { - callback.onSuccess(); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); } } catch (Throwable e) { callback.onFailure(e); @@ -124,6 +129,13 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } + private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { if (statsEnabled) { stats.log(toDeviceActorMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index 5b94db28b0..d7eebd98b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -36,6 +36,7 @@ public class MsgPackCallback i @Override public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); if (ackMap.remove(id) != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } @@ -43,6 +44,7 @@ public class MsgPackCallback i @Override public void onFailure(Throwable t) { + log.trace("[{}] ON FAILURE", id); TbProtoQueueMsg message = ackMap.remove(id); log.warn("Failed to process message: {}", message.getValue(), t); if (ackMap.isEmpty()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index f7912836c5..f2956ea747 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -57,6 +57,10 @@ public class TbCoreConsumerStats { } } + public void log(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg) { + //TODO 2.5 + } + public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { @@ -67,4 +71,5 @@ public class TbCoreConsumerStats { subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 31b712644d..8d82b90fb6 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -33,6 +33,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -50,13 +51,19 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.PartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -67,7 +74,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -118,6 +124,12 @@ public class DefaultDeviceStateService implements DeviceStateService { @Lazy private ActorService actorService; + @Autowired + private TbCoreQueueProvider queueProvider; + + @Autowired + private PartitionService partitionService; + @Autowired private TelemetrySubscriptionService tsSubService; @@ -151,7 +163,6 @@ public class DefaultDeviceStateService implements DeviceStateService { queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); - //TODO: schedule persistence in v2.1; } @PreDestroy @@ -163,38 +174,79 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - queueExecutor.submit(() -> onDeviceAddedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override public void onDeviceUpdated(Device device) { - queueExecutor.submit(() -> onDeviceUpdatedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); + } + + @Override + public void onDeviceDeleted(Device device) { + sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); } @Override public void onDeviceConnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } } @Override public void onDeviceActivity(DeviceId deviceId) { deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); + long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } } @Override public void onDeviceDisconnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); - } - - @Override - public void onDeviceDeleted(Device device) { - queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } } @Override @@ -206,26 +258,56 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.DeviceStateServiceMsgProto proto; + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { try { - proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - onDeviceAdded(device); - } else if (proto.getUpdated()) { - onDeviceUpdated(device); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + onDeviceDeleted(tenantId, deviceId); + callback.onSuccess(); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + addDeviceUsingState(state); + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + callback.onFailure(t); + } + }); + } else if (proto.getUpdated()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } + } else { + //Device was probably deleted while message was in queue; + callback.onSuccess(); } } + + } catch (Exception e) { + log.trace("Failed to process queue msg: [{}]", proto, e); + callback.onFailure(e); + } + + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + repartition(partitionChangeEvent.getPartitions()); } } @@ -241,9 +323,9 @@ public class DefaultDeviceStateService implements DeviceStateService { for (Device device : page.getData()) { //TODO 2.5 // if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } + if (!deviceStates.containsKey(device.getId())) { + fetchFutures.add(fetchDeviceState(device)); + } // } else { // Set tenantDeviceSet = tenantDevices.get(tenant.getId()); // if (tenantDeviceSet != null) { @@ -275,7 +357,7 @@ public class DefaultDeviceStateService implements DeviceStateService { for (Device device : page.getData()) { //TODO 2.5 // if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); + fetchFutures.add(fetchDeviceState(device)); // } } try { @@ -319,105 +401,29 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void onDeviceConnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } - } - - private void onDeviceDisconnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } - } - - private void onDeviceActivitySync(DeviceId deviceId) { - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - } - private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { //TODO 2.5 // if (!routingService.resolveById(deviceId).isPresent()) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); - } + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); } + } // } } return deviceStateData; } - private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - - private void onDeviceAddedSync(Device device) { - //TODO 2.5 -// Optional address = routingService.resolveById(device.getId()); -// if (!address.isPresent()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - } - }); -// } else { -// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); -// } - } - - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { - log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); - ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + log.trace("[{}][{}] Device is monitored on partition: {}", tenantId, deviceId, tpi); + TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -425,43 +431,22 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - //TODO 2.5 -// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); - } - - private void onDeviceUpdatedSync(Device device) { - //TODO 2.5 -// Optional address = routingService.resolveById(device.getId()); -// if (!address.isPresent()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } -// } else { -// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); -// } + TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); + queueProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - //TODO 2.5 -// Optional address = routingService.resolveById(deviceId); -// if (!address.isPresent()) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + Set deviceIds = tenantDevices.get(tenantId); + if (deviceIds != null) { + deviceIds.remove(deviceId); + if (deviceIds.isEmpty()) { + tenantDevices.remove(tenantId); } -// } else { -// sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); -// } + } } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 8dbe7579e3..636a9e9bb1 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.service.state; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.service.queue.TbMsgCallback; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService { +public interface DeviceStateService extends ApplicationListener { void onDeviceAdded(Device device); @@ -38,7 +41,6 @@ public interface DeviceStateService { void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout); - void onClusterUpdate(); + void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto serverAddress, TbMsgCallback bytes); - void onRemoteMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto index cfacc66121..46526e8e1c 100644 --- a/application/src/main/proto/cluster.proto +++ b/application/src/main/proto/cluster.proto @@ -134,13 +134,3 @@ message FromDeviceRPCResponseProto { string response = 3; int32 error = 4; } - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e147325f42..3d47db432f 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index b70fc417c8..a6193a0b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -46,16 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - private static ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); - protected static byte[] longToBytes(long x) { + ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } protected static long bytesToLong(byte[] bytes) { - longBuffer.put(bytes, 0, bytes.length); - longBuffer.flip();//need flip - return longBuffer.getLong(); + return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java index 25c936f829..831748da06 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java @@ -101,6 +101,7 @@ public class ConsistentHashPartitionService implements PartitionService { return topicPartitions; } + //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java index 6fe4571517..8272df5fdc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.discovery; -public interface PartitionDiscoveryService { +public interface DiscoveryService { } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java index 8983cf1e97..28fd5c1fb3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -28,7 +28,7 @@ import java.util.Collections; @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") -public class DummyDiscoveryService implements PartitionDiscoveryService { +public class DummyDiscoveryService implements DiscoveryService { private final TbServiceInfoProvider serviceInfoProvider; private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java index 07f11617ac..48c886b7e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.discovery; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -31,7 +30,6 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -39,18 +37,12 @@ import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -60,7 +52,7 @@ import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent. @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j -public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, PathChildrenCacheListener { +public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener { @Value("${zk.url}") private String zkUrl; @@ -84,7 +76,7 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P private volatile boolean stopped = true; - public ZkPartitionDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { this.serviceInfoProvider = serviceInfoProvider; this.partitionService = partitionService; } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index 01e7a211d3..b7412a7c99 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private TbQueueProducer> tbCoreProducer; + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings, @@ -77,13 +79,21 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return requestBuilder.build(); } + //TODO 2.5 Singleton @Override public TbQueueProducer> getTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - return requestBuilder.build(); + if (tbCoreProducer == null) { + synchronized (this) { + if (tbCoreProducer == null) { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + tbCoreProducer = requestBuilder.build(); + } + } + } + return tbCoreProducer; } @Override diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/queue.proto similarity index 95% rename from common/queue/src/main/proto/transport.proto rename to common/queue/src/main/proto/queue.proto index 8c564b6540..17f8b9c31c 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/queue.proto @@ -239,6 +239,20 @@ message DeviceActorToTransportMsg { ToServerRpcResponseMsg toServerResponse = 7; } +/** + * TB Core to TB Core messages + */ + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + /** * Main messages; */ @@ -259,6 +273,7 @@ message TransportApiResponseMsg { /* Messages that are handled by ThingsBoard Core Service */ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; + DeviceStateServiceMsgProto deviceStateServiceMsg = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From 7203e7aa19a208a850c45149f14c0be348b53677 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 Mar 2020 18:07:24 +0200 Subject: [PATCH 017/102] Distributed Mode fixes --- .../server/actors/ActorSystemContext.java | 9 +- .../actors/ruleChain/DefaultTbContext.java | 2 +- .../server/actors/stats/StatsActor.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 12 +- .../DefaultTbRuleEngineConsumerService.java | 10 +- .../server/service/queue/MsgPackCallback.java | 2 +- .../service/queue/TbCoreConsumerService.java | 5 +- .../queue/TbRuleEngineConsumerService.java | 2 +- .../service/script/RemoteJsInvokeService.java | 6 +- .../script/RemoteJsRequestEncoder.java | 4 +- .../script/RemoteJsResponseDecoder.java | 6 +- .../state/DefaultDeviceStateService.java | 129 +++++----- .../service/state/DeviceStateService.java | 2 +- .../DefaultTbCoreToTransportService.java | 12 +- .../transport/DefaultTransportApiService.java | 2 +- .../transport/RemoteTransportApiService.java | 18 +- .../transport/ToRuleEngineMsgDecoder.java | 4 +- .../transport/ToTransportMsgEncoder.java | 2 +- .../transport/TransportApiService.java | 4 +- .../ConsistentHashParitionServiceTest.java | 8 +- .../server/discovery/TopicPartitionInfo.java | 41 ---- .../server/kafka/TbKafkaProperty.java | 32 --- .../server/kafka/TbNodeIdProvider.java | 51 ---- .../server/{ => queue}/TbQueueAdmin.java | 2 +- .../server/{ => queue}/TbQueueCallback.java | 2 +- .../server/{ => queue}/TbQueueConsumer.java | 7 +- .../{ => queue}/TbQueueCoreSettings.java | 2 +- .../server/{ => queue}/TbQueueHandler.java | 2 +- .../server/{ => queue}/TbQueueMsg.java | 2 +- .../server/{ => queue}/TbQueueMsgHeaders.java | 2 +- .../{ => queue}/TbQueueMsgMetadata.java | 2 +- .../server/{ => queue}/TbQueueProducer.java | 4 +- .../{ => queue}/TbQueueRequestTemplate.java | 2 +- .../{ => queue}/TbQueueResponseTemplate.java | 2 +- .../TbQueueRuleEngineSettings.java | 2 +- .../TbQueueTransportApiSettings.java | 2 +- .../TbQueueTransportNotificationSettings.java | 2 +- .../common/AbstractTbQueueTemplate.java | 4 +- .../common/AsyncCallbackTemplate.java | 2 +- .../common/DefaultTbQueueMsgHeaders.java | 4 +- .../common/DefaultTbQueueRequestTemplate.java | 18 +- .../DefaultTbQueueResponseTemplate.java | 14 +- .../{ => queue}/common/TbProtoQueueMsg.java | 6 +- .../discovery/ConsistentHashCircle.java | 2 +- .../ConsistentHashPartitionService.java | 20 +- .../DefaultTbServiceInfoProvider.java | 4 +- .../discovery/DiscoveryService.java | 2 +- .../discovery/DummyDiscoveryService.java | 2 +- .../discovery/PartitionChangeEvent.java | 9 +- .../discovery/PartitionService.java | 10 +- .../{ => queue}/discovery/ServiceKey.java | 2 +- .../{ => queue}/discovery/ServiceType.java | 8 +- .../discovery/TbServiceInfoProvider.java | 2 +- .../queue/discovery/TopicPartitionInfo.java | 78 ++++++ .../discovery/ZkDiscoveryService.java | 2 +- .../environment/EnvironmentLogService.java | 2 +- .../{ => queue}/kafka/KafkaTbQueueMsg.java | 8 +- .../kafka/KafkaTbQueueMsgMetadata.java | 4 +- .../{ => queue}/kafka/TBKafkaAdmin.java | 7 +- .../kafka/TBKafkaConsumerTemplate.java | 14 +- .../kafka/TBKafkaProducerTemplate.java | 26 +- .../{ => queue}/kafka/TbKafkaDecoder.java | 4 +- .../{ => queue}/kafka/TbKafkaEncoder.java | 2 +- .../{ => queue}/kafka/TbKafkaHandler.java | 2 +- .../{ => queue}/kafka/TbKafkaPartitioner.java | 2 +- .../server/queue/kafka/TbKafkaProperty.java | 28 +++ .../{ => queue}/kafka/TbKafkaSettings.java | 3 +- .../{ => queue}/memory/InMemoryStorage.java | 6 +- .../memory/InMemoryTbQueueConsumer.java | 10 +- .../memory/InMemoryTbQueueProducer.java | 10 +- .../InMemoryMonolithQueueProvider.java | 21 +- .../InMemoryTransportQueueProvider.java | 28 +-- .../provider/KafkaMonolithQueueProvider.java | 52 ++-- .../provider/KafkaTbCoreQueueProvider.java | 82 ++++--- .../KafkaTbRuleEngineQueueProvider.java | 63 ++--- .../provider/KafkaTransportQueueProvider.java | 58 ++--- .../provider/TbCoreQueueProvider.java | 10 +- .../provider/TbRuleEngineQueueProvider.java | 10 +- .../provider/TransportQueueProvider.java | 10 +- .../transport/coap/CoapTransportContext.java | 2 +- .../transport/coap/CoapTransportService.java | 2 +- .../transport/http/DeviceApiController.java | 2 +- .../transport/http/HttpTransportContext.java | 2 +- .../transport/mqtt/MqttTransportContext.java | 12 +- .../transport/mqtt/MqttTransportService.java | 2 +- .../common/transport/TransportContext.java | 13 +- .../service/DefaultTransportService.java | 25 +- .../service/ToRuleEngineMsgEncoder.java | 2 +- .../ToTransportMsgResponseDecoder.java | 4 +- .../service/TransportApiRequestEncoder.java | 2 +- .../service/TransportApiResponseDecoder.java | 4 +- .../ThingsboardCoapTransportApplication.java | 2 +- .../ThingsboardHttpTransportApplication.java | 2 +- .../ThingsboardMqttTransportApplication.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 36 ++- ui/package-lock.json | 225 ++++++++---------- 96 files changed, 690 insertions(+), 700 deletions(-) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueAdmin.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueCallback.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueConsumer.java (82%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueCoreSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueHandler.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsg.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsgHeaders.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsgMetadata.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueProducer.java (88%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueRequestTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueResponseTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueRuleEngineSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueTransportApiSettings.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueTransportNotificationSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/AbstractTbQueueTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/AsyncCallbackTemplate.java (98%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueMsgHeaders.java (91%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueRequestTemplate.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueResponseTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/TbProtoQueueMsg.java (90%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ConsistentHashCircle.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ConsistentHashPartitionService.java (92%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DefaultTbServiceInfoProvider.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DiscoveryService.java (93%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DummyDiscoveryService.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/PartitionChangeEvent.java (85%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/PartitionService.java (68%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ServiceKey.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ServiceType.java (72%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/TbServiceInfoProvider.java (94%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ZkDiscoveryService.java (99%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/environment/EnvironmentLogService.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/KafkaTbQueueMsg.java (87%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/KafkaTbQueueMsgMetadata.java (89%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaAdmin.java (93%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaConsumerTemplate.java (90%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaProducerTemplate.java (76%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaDecoder.java (89%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaEncoder.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaHandler.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaPartitioner.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaSettings.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryStorage.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryTbQueueConsumer.java (81%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryTbQueueProducer.java (84%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/InMemoryMonolithQueueProvider.java (85%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/InMemoryTransportQueueProvider.java (81%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaMonolithQueueProvider.java (75%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTbCoreQueueProvider.java (52%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTbRuleEngineQueueProvider.java (53%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTransportQueueProvider.java (72%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TbCoreQueueProvider.java (88%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TbRuleEngineQueueProvider.java (83%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TransportQueueProvider.java (84%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 6b8e950fdb..46e6df60ab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -35,7 +35,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; @@ -46,7 +45,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; @@ -65,8 +63,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.discovery.TbServiceInfoProvider; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -321,10 +318,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Autowired - @Getter - private TbNodeIdProvider nodeIdProvider; - @Getter @Setter private ActorRef appActor; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 3e982b520e..2de491c89d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -254,7 +254,7 @@ class DefaultTbContext implements TbContext { @Override public String getNodeId() { - return mainCtx.getNodeIdProvider().getNodeId(); + return mainCtx.getServiceInfoProvider().getServiceId(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 8757deb2b4..1ef33467b2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c95599f2ad..07b7704d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,15 +24,15 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 253c7d9933..6d06b9ee11 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -23,13 +23,13 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d7eebd98b0..d275adc4be 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; import java.util.concurrent.ConcurrentMap; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 8a39b5df4a..2dcdb0e0c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -16,10 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.function.Consumer; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; public interface TbCoreConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 968fbb2274..19966b4566 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; public interface TbRuleEngineConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 08578c3890..8150a77c30 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -24,9 +24,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import javax.annotation.Nullable; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index 1b33fd85cc..51ecba27b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,9 +17,9 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index ef15d5c877..621d2b05d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,10 +16,10 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 8d82b90fb6..4e395f7090 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +32,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -56,13 +55,12 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.PartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TopicPartitionInfo; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -153,6 +151,7 @@ public class DefaultDeviceStateService implements DeviceStateService { private ListeningScheduledExecutorService queueExecutor; private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); @@ -161,7 +160,6 @@ public class DefaultDeviceStateService implements DeviceStateService { public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); - queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); } @@ -249,14 +247,6 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - @Override - public void onClusterUpdate() { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(this::onClusterUpdateSync); - } - } - @Override public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { try { @@ -272,8 +262,15 @@ public class DefaultDeviceStateService implements DeviceStateService { Futures.addCallback(fetchDeviceState(device), new FutureCallback() { @Override public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - callback.onSuccess(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); + if (partitionedDevices.containsKey(tpi)) { + addDeviceUsingState(tpi, state); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , tenantId, deviceId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); + } } @Override @@ -307,61 +304,63 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { - repartition(partitionChangeEvent.getPartitions()); - } - } - - private void onClusterUpdateSync() { - clusterUpdatePending = false; - List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); - for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); - TextPageLink pageLink = new TextPageLink(initFetchPackSize); - while (pageLink != null) { - TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); - pageLink = page.getNextPageLink(); - for (Device device : page.getData()) { - //TODO 2.5 -// if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } -// } else { -// Set tenantDeviceSet = tenantDevices.get(tenant.getId()); -// if (tenantDeviceSet != null) { -// tenantDeviceSet.remove(device.getId()); -// } -// deviceStates.remove(device.getId()); -// deviceLastReportedActivity.remove(device.getId()); -// deviceLastSavedActivity.remove(device.getId()); -// } - } - try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); - } catch (InterruptedException | ExecutionException e) { - log.warn("Failed to init device state service from DB", e); + synchronized (this) { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(() -> { + clusterUpdatePending = false; + initStateFromDB(partitionChangeEvent.getPartitions()); + }); } } } } - private void initStateFromDB() { + private void initStateFromDB(Set partitions) { try { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(partitionedDevices.keySet()); + + Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); + removedPartitions.removeAll(partitions); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set devices = partitionedDevices.remove(partition); + devices.forEach(deviceId -> { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + }); + }); + + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. + // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(initFetchPackSize); while (pageLink != null) { + List> fetchFutures = new ArrayList<>(); TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - //TODO 2.5 -// if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); -// } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); + if (addedPartitions.contains(tpi)) { + ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { + @Nullable + @Override + public Void apply(@Nullable DeviceStateData state) { + if (state != null) { + addDeviceUsingState(tpi, state); + } + return null; + } + }); + fetchFutures.add(future); + } } try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + Futures.successfulAsList(fetchFutures).get(); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } @@ -372,11 +371,12 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void addDeviceUsingState(DeviceStateData state) { - tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { + partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } + //TODO 2.5: review this method private void updateState() { long ts = System.currentTimeMillis(); Set deviceIds = new HashSet<>(deviceStates.keySet()); @@ -404,8 +404,6 @@ public class DefaultDeviceStateService implements DeviceStateService { private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - //TODO 2.5 -// if (!routingService.resolveById(deviceId).isPresent()) { Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); if (device != null) { try { @@ -415,7 +413,6 @@ public class DefaultDeviceStateService implements DeviceStateService { log.debug("[{}] Failed to fetch device state!", deviceId, e); } } -// } } return deviceStateData; } @@ -538,7 +535,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private class AttributeSaveCallback implements FutureCallback { + private static class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 636a9e9bb1..14a25c2e9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.state; import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.queue.TbMsgCallback; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 833b28c3b2..fb369f4564 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -19,14 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import java.util.UUID; import java.util.function.Consumer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 51597877b4..42ee4c4619 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index e84e1b595e..4c9c801e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -16,24 +16,18 @@ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueResponseTemplate; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.DefaultTbQueueResponseTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index bdebacbd13..71739c0439 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java index 9c3f5634bf..50b940ae48 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index eab2e07b0e..c9edfa9331 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.TbQueueHandler; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 9442202688..445465f5df 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -25,10 +25,10 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TbServiceInfoProvider; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.ArrayList; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java deleted file mode 100644 index 4480d07dd6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.discovery; - -import lombok.Builder; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Optional; - -@Builder -public class TopicPartitionInfo { - - private final String topic; - private final TenantId tenantId; - private final Integer partition; - - public String getTopic() { - return topic; - } - - public Optional getTenantId() { - return Optional.ofNullable(tenantId); - } - - public Optional getPartition() { - return Optional.ofNullable(partition); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java deleted file mode 100644 index cd52948da8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -/** - * Created by ashvayka on 25.09.18. - */ -@Data -public class TbKafkaProperty { - - private String key; - private String value; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java deleted file mode 100644 index 343c5e6dd8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Component -public class TbNodeIdProvider { - - @Getter - @Value("${cluster.node_id:#{null}}") - private String nodeId; - - @PostConstruct - public void init() { - if (StringUtils.isEmpty(nodeId)) { - try { - nodeId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current NodeId: {}", nodeId); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 5952e7da70..deb84f440e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueAdmin { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index e823d2a5fc..f21ad80d10 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java similarity index 82% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index ac4d7611f6..1a5c98ec80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -13,9 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; + +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; +import java.util.Set; public interface TbQueueConsumer { @@ -23,7 +26,7 @@ public interface TbQueueConsumer { void subscribe(); - void subscribe(List partitions); + void subscribe(Set partitions); void unsubscribe(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java index dc84625634..4957b5ed5b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java index 93004cddca..609cbd2107 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java index ca8f3a61da..b0e4990305 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java index f95454c976..f205ed676f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java index a3331999a5..0c24fec438 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueMsgMetadata { } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index 2498117c46..e7ea35b836 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; public interface TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 2eb429b082..20530360d6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index 686570ca32..c823b3df88 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java index 4acfec1091..fc3a36a88a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java index b98bd3b76d..263cf262a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java index 2666045e7e..0f2ac55862 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index a6193a0b48..ec3b0b537a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java index 95783f87cd..d06645abd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java index de0c368a65..36c3dd0999 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; -import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.HashMap; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 0388daebb6..a2e92e112b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -21,14 +21,14 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.InterruptException; -import org.thingsboard.server.TbQueueAdmin; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index c77692ee36..3e70040e2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.InterruptException; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueHandler; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueResponseTemplate; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 8dd492504c..8823002530 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import lombok.Data; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java index b4c382050b..1b40545cde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java similarity index 92% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 831748da06..97b660a48c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import com.google.common.hash.HashCode; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -40,6 +39,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; +import java.util.stream.Collectors; @Service @Slf4j @@ -90,6 +90,7 @@ public class ConsistentHashPartitionService implements PartitionService { List partitions = myPartitions.get(serviceKey); List topicPartitions = new ArrayList<>(); for (Integer partition : partitions) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -104,11 +105,19 @@ public class ConsistentHashPartitionService implements PartitionService { //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); int partition = Math.abs(hash % partitionSizes.get(serviceType)); + return buildTopicPartitionInfo(serviceType, tenantId, partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { + return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -153,7 +162,10 @@ public class ConsistentHashPartitionService implements PartitionService { myPartitions.forEach((serviceKey, partitions) -> { if (!partitions.equals(oldPartitions.get(serviceKey))) { log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, partitions)); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) + .collect(Collectors.toSet()); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } }); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 80040a017b..cb7b04df44 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -64,7 +64,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (serviceType.equalsIgnoreCase("monolith")) { serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); } else { - serviceTypes = Collections.singletonList(ServiceType.valueOf(serviceType)); + serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } ServiceInfo.Builder builder = ServiceInfo.newBuilder() .setServiceId(serviceId) diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java index 8272df5fdc..36510261d6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; public interface DiscoveryService { diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 28fd5c1fb3..9017823efb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 570a774c9c..13976232e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; -import lombok.Setter; import org.springframework.context.ApplicationEvent; -import java.util.List; +import java.util.Set; public class PartitionChangeEvent extends ApplicationEvent { @@ -27,9 +26,9 @@ public class PartitionChangeEvent extends ApplicationEvent { @Getter private final ServiceKey serviceKey; @Getter - private final List partitions; + private final Set partitions; - public PartitionChangeEvent(Object source, ServiceKey serviceKey, List partitions) { + public PartitionChangeEvent(Object source, ServiceKey serviceKey, Set partitions) { super(source); this.serviceKey = serviceKey; this.partitions = partitions; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java similarity index 68% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index ad61a63807..f8613f5fb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -21,11 +21,19 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; +/** + * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} + */ public interface PartitionService { List getCurrentPartitions(ServiceType serviceType); TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + /** + * Received from the Discovery service when network topology is changed. + * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java index 0179be968c..3396078124 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java similarity index 72% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java index 801069eb61..a5de33c436 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java @@ -13,8 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + + public static ServiceType of(String serviceType) { + return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 5cb545c5d6..764f646a8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java new file mode 100644 index 0000000000..05c5c1586d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Builder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; +import java.util.Optional; + +public class TopicPartitionInfo { + + private final String topic; + private final TenantId tenantId; + private final Integer partition; + private final String fullTopicName; + + @Builder + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition) { + this.topic = topic; + this.tenantId = tenantId; + this.partition = partition; + String tmp = topic; + if (tenantId != null) { + tmp += "." + tenantId.getId().toString(); + } + if (partition != null) { + tmp += "." + partition; + } + + this.fullTopicName = tmp; + } + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + + public Optional getPartition() { + return Optional.ofNullable(partition); + } + + public String getFullTopicName() { + return fullTopicName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfo that = (TopicPartitionInfo) o; + return topic.equals(that.topic) && + Objects.equals(tenantId, that.tenantId) && + Objects.equals(partition, that.partition) && + fullTopicName.equals(that.fullTopicName); + } + + @Override + public int hashCode() { + return Objects.hash(fullTopicName); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java similarity index 99% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 48c886b7e4..e761a229c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java index cff3fd1056..267b34b204 100644 --- a/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.environment; +package org.thingsboard.server.queue.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java similarity index 87% rename from common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index 9f8e73bdfc..98470eafba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgHeaders; -import org.thingsboard.server.common.DefaultTbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java similarity index 89% rename from common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java index 09cc292deb..5a9eaa632c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.kafka.clients.producer.RecordMetadata; -import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueMsgMetadata; @Data @AllArgsConstructor diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index 5f1b41ea7b..e1ca580c28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.JdkFutureAdapters; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; @@ -25,7 +22,7 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.errors.TopicExistsException; -import org.thingsboard.server.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueAdmin; import java.util.Collections; import java.util.concurrent.ExecutionException; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index eb35c583c8..b4b1578939 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.Getter; @@ -22,8 +22,9 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.io.IOException; import java.time.Duration; @@ -31,7 +32,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.concurrent.ExecutionException; +import java.util.Set; import java.util.stream.Collectors; /** @@ -79,9 +80,10 @@ public class TBKafkaConsumerTemplate implements TbQueueCon } @Override - public void subscribe(List partitions) { - List topicNames = partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList()); + public void subscribe(Set partitions) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); topicNames.forEach(this::createTopicIfNotExists); + consumer.unsubscribe(); consumer.subscribe(topicNames); subscribed = true; } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 4999ba026a..8f177ecb3f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.Getter; @@ -21,22 +21,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import java.util.List; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -76,14 +69,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro byte[] data = msg.getData(); ProducerRecord record; Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - StringBuilder topic = new StringBuilder().append(tpi.getTopic()); - if (tpi.getTenantId().isPresent()) { - topic.append(".").append(tpi.getTenantId().get().getId().toString()); - } - if (tpi.getPartition().isPresent()) { - topic.append(".").append(tpi.getPartition().get()); - } - record = new ProducerRecord<>(topic.toString(), null, key, data, headers); + record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java similarity index 89% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java index 564d905675..6e3ea1e4a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import java.io.IOException; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java index 22f65c58a0..b3c4dec8ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java index 5e5aa0591a..8c2cbd2771 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java index 8ec5cd1d97..b31f87642b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import org.apache.kafka.clients.producer.Partitioner; import org.apache.kafka.common.PartitionInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java new file mode 100644 index 0000000000..b6de7db50d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Data; + +/** + * Created by ashvayka on 25.09.18. + */ +@Data +public class TbKafkaProperty { + + private String key; + private String value; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 912e935bef..e8581d9719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 86d103ea70..a61bf560ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -13,17 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 3f1e844efc..c0849d083e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; +import java.util.Set; public class InMemoryTbQueueConsumer implements TbQueueConsumer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); @@ -40,7 +42,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon } @Override - public void subscribe(List partitions) { + public void subscribe(Set partitions) { } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index 79fa1d6b39..c235f876ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; import lombok.Data; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java index 7fc7bdea7c..b734562a61 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java @@ -13,26 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Slf4j @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java index 2d7205e516..3e33e4daa6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java @@ -13,33 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueAdmin; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='in-memory'") +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class InMemoryTransportQueueProvider implements TransportQueueProvider { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index b7412a7c99..990baca5a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,33 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; @@ -48,13 +48,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private TbQueueProducer> tbCoreProducer; public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, - TbNodeIdProvider nodeIdProvider, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -65,7 +65,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-notifications" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); return requestBuilder.build(); } @@ -74,7 +74,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -87,7 +87,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn if (tbCoreProducer == null) { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); tbCoreProducer = requestBuilder.build(); } @@ -101,8 +101,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -112,8 +112,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -123,8 +123,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("transport-api-consumer-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -133,7 +133,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-producer-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java similarity index 52% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 72338bb149..0d6d62c3cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -13,43 +13,59 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + private TbQueueProducer> tbCoreProducer; + + public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,18 +74,25 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - return requestBuilder.build(); + if (tbCoreProducer == null) { + synchronized (this) { + if (tbCoreProducer == null) { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + tbCoreProducer = requestBuilder.build(); + } + } + } + return tbCoreProducer; } @Override @@ -77,29 +100,28 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java similarity index 53% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index d635419141..c75437f38b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -13,43 +13,54 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } - @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,29 +69,27 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } - @Override public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java similarity index 72% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java index 577c9180fc..10cc9c1871 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java @@ -13,30 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -44,20 +44,20 @@ import org.thingsboard.server.kafka.TbNodeIdProvider; public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbNodeIdProvider nodeIdProvider, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -68,14 +68,14 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-node-" + nodeIdProvider.getNodeId()); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder @@ -93,8 +93,8 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -102,8 +102,8 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-tb-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -111,9 +111,9 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueConsumer> getTransportNotificationsConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index ba93bf96ef..9fdec01830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -13,13 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java index b53c3ac28b..c4d610cab1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java @@ -13,16 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java index de69e1819a..2aa18a62ca 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 5be2f76258..864e7b2ab9 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index d725d47a5f..9323cfa7a0 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -36,7 +36,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 9df658224a..c8e7f76450 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 3d344d8dc5..28c38bdda6 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 107639f302..b058a1c260 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,33 +15,23 @@ */ package org.thingsboard.server.transport.mqtt; -import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 8f752b5066..fb8a0b70be 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index ab67beee0b..c49b9c5bd3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,7 +20,9 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -32,15 +34,16 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -public class TransportContext { +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") +public abstract class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; - @Autowired - private TbNodeIdProvider nodeIdProvider; + private TbServiceInfoProvider serviceInfoProvider; @Getter private ExecutorService executor; @@ -58,7 +61,7 @@ public class TransportContext { } public String getNodeId() { - return nodeIdProvider.getNodeId(); + return serviceInfoProvider.getServiceId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index f1b73f30af..fcb630d041 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -16,16 +16,16 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,10 +34,10 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.discovery.PartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TopicPartitionInfo; -import org.thingsboard.server.provider.TransportQueueProvider; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -46,7 +46,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; -import org.thingsboard.server.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -66,6 +66,7 @@ import java.util.concurrent.TimeUnit; */ @Slf4j @Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") public class DefaultTransportService implements TransportService { @Value("${transport.rate_limits.enabled}") diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index 42a65440a0..fc1445539b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index 31a32d8368..cc154f87b2 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 6c4b362f03..3971fdbf83 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index 89d61fba56..5fdc58c068 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index 6dbca324e1..ec596db10d 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index 39dc0677e8..f44af2105e 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index ce95ee6f1f..cd7225ad1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 721db2605d..f4a31449e7 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -62,11 +62,8 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -service: - type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport - queue: - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka or ? kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -74,18 +71,39 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" - notifications: - topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + transport: + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/ui/package-lock.json b/ui/package-lock.json index dccf637726..512bac811c 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" + "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" + "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", + "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", + "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", + "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", + "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", + "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", + "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", "dev": true }, "attr-accept": { @@ -2174,7 +2174,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2198,7 +2198,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2207,7 +2207,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2216,7 +2216,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2362,7 +2362,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2586,7 +2586,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2617,7 +2617,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2794,7 +2794,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3121,7 +3121,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3181,7 +3181,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", "dev": true }, "convert-source-map": { @@ -3208,7 +3208,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3427,7 +3427,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", + "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3555,7 +3555,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "dev": true, "requires": { "ms": "2.0.0" @@ -3633,7 +3633,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3643,7 +3643,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3652,7 +3652,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3661,7 +3661,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3724,7 +3724,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" }, "delegates": { "version": "1.0.0", @@ -3880,7 +3880,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", "dev": true }, "domelementtype": { @@ -4035,7 +4035,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", "dev": true, "requires": { "prr": "~1.0.1" @@ -4554,7 +4554,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4769,7 +4769,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5197,7 +5197,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", + "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", "dev": true }, "fs-write-stream-atomic": { @@ -5238,8 +5238,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5260,14 +5259,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5282,20 +5279,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5412,8 +5406,7 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5425,7 +5418,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5440,7 +5432,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5448,14 +5439,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5474,7 +5463,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5564,8 +5552,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5577,7 +5564,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5663,8 +5649,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5700,7 +5685,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5720,7 +5704,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5764,14 +5747,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -5790,7 +5771,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", "dev": true }, "functional-red-black-tree": { @@ -6738,7 +6719,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6788,7 +6769,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6890,7 +6871,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", "dev": true }, "is-callable": { @@ -6934,7 +6915,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6945,7 +6926,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", "dev": true } } @@ -7053,7 +7034,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7120,7 +7101,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", "dev": true }, "is-word-character": { @@ -7260,7 +7241,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", "dev": true }, "json-schema": { @@ -7277,7 +7258,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7517,7 +7498,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8050,7 +8031,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" + "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" }, "methods": { "version": "1.1.2", @@ -8092,7 +8073,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", "dev": true }, "mime-db": { @@ -8113,7 +8094,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" }, "min-document": { "version": "2.19.0", @@ -8170,7 +8151,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", "requires": { "brace-expansion": "^1.1.7" } @@ -8265,7 +8246,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8343,7 +8324,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8495,7 +8476,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", + "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8555,7 +8536,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8574,7 +8555,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8775,7 +8756,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9092,7 +9073,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9520,7 +9501,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9823,7 +9804,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", "dev": true }, "process": { @@ -9846,7 +9827,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", "requires": { "asap": "~2.0.3" } @@ -9926,7 +9907,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -10091,7 +10072,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", + "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -10122,7 +10103,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", + "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10281,7 +10262,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10293,7 +10274,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", + "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", "requires": { "lodash": "^4.0.1" } @@ -10476,7 +10457,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" }, "regenerator-transform": { "version": "0.14.1", @@ -10490,7 +10471,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10795,7 +10776,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", "dev": true }, "retry": { @@ -10899,7 +10880,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sass-graph": { "version": "2.2.4", @@ -11233,7 +11214,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", "dev": true, "requires": { "base": "^0.11.1", @@ -11269,7 +11250,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11289,7 +11270,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11298,7 +11279,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11307,7 +11288,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11320,7 +11301,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11340,7 +11321,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", + "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11464,7 +11445,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11558,7 +11539,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11698,7 +11679,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11742,7 +11723,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "requires": { "safe-buffer": "~5.1.0" } @@ -12711,7 +12692,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" + "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" }, "table": { "version": "5.4.6", @@ -12941,7 +12922,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", "requires": { "os-tmpdir": "~1.0.2" } @@ -12981,7 +12962,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", "dev": true, "requires": { "define-property": "^2.0.2", @@ -13183,7 +13164,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", "dev": true } } @@ -13508,7 +13489,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13570,7 +13551,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", "dev": true }, "util": { @@ -14309,7 +14290,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", + "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", "dev": true }, "whatwg-fetch": { @@ -14423,7 +14404,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", + "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" From e102b55bc139deff72c554f8e7f6e7c9f0c029a4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 Mar 2020 18:39:34 +0200 Subject: [PATCH 018/102] Thread-safety usage of Kafka Consumer --- .../state/DefaultDeviceStateService.java | 8 ++++--- .../queue/kafka/TBKafkaConsumerTemplate.java | 23 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 4e395f7090..5ee26bc294 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -293,12 +293,10 @@ public class DefaultDeviceStateService implements DeviceStateService { callback.onSuccess(); } } - } catch (Exception e) { log.trace("Failed to process queue msg: [{}]", proto, e); callback.onFailure(e); } - } @Override @@ -366,6 +364,10 @@ public class DefaultDeviceStateService implements DeviceStateService { } } } + log.info("Managing following partitions:"); + partitionedDevices.forEach((tpi, devices) -> { + log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); + }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index b4b1578939..09c29aa5e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; + private volatile Set partitions; @Getter private final String topic; @@ -74,29 +75,31 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { - createTopicIfNotExists(topic); - consumer.subscribe(Collections.singletonList(topic)); - subscribed = true; + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null)); + subscribed = false; } @Override public void subscribe(Set partitions) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(this::createTopicIfNotExists); - consumer.unsubscribe(); - consumer.subscribe(topicNames); - subscribed = true; + this.partitions = partitions; + subscribed = false; } @Override public List poll(long durationInMillis) { - if (!subscribed) { + if (!subscribed && partitions == null) { try { Thread.sleep(durationInMillis); } catch (InterruptedException e) { log.debug("Failed to await subscription", e); } } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(this::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); if (records.count() > 0) { List result = new ArrayList<>(); From ce6ec889833db0802fa3b7af01b02ae95873c754 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 20 Mar 2020 18:00:57 +0200 Subject: [PATCH 019/102] Refactoring of the websocket and subscription services --- .../device/DeviceActorMessageProcessor.java | 3 +- .../server/actors/service/ActorService.java | 3 +- .../actors/service/DefaultActorService.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 84 ++- .../server/service/queue/TbMsgCallback.java | 13 + .../state/DefaultDeviceStateService.java | 15 +- .../DefaultLocalSubscriptionService.java | 230 ++++++ .../DefaultSubscriptionManagerService.java | 377 ++++++++++ .../LocalSubscriptionService.java | 36 + .../SubscriptionManagerService.java | 37 + .../subscription/TbAttributeSubscription.java | 50 ++ .../TbAttributeSubscriptionScope.java | 22 + .../service/subscription/TbSubscription.java | 52 ++ .../subscription/TbSubscriptionType.java | 20 + .../subscription/TbSubscriptionUtils.java | 247 +++++++ .../TbTimeseriesSubscription.java | 51 ++ .../DefaultTelemetrySubscriptionService.java | 657 ++---------------- .../DefaultTelemetryWebSocketService.java | 72 +- .../TelemetrySubscriptionService.java | 23 +- .../telemetry/sub/SubscriptionUpdate.java | 2 +- application/src/main/proto/cluster.proto | 62 -- .../src/main/resources/thingsboard.yml | 2 +- .../common/data/kv/BaseAttributeKvEntry.java | 4 + .../discovery/ClusterTopologyChangeEvent.java | 33 + .../ConsistentHashPartitionService.java | 119 +++- .../queue/discovery/PartitionService.java | 9 + .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- common/queue/src/main/proto/queue.proto | 97 ++- .../service/DefaultTransportService.java | 1 + .../api/RuleEngineTelemetryService.java | 2 - .../engine/telemetry/TbMsgAttributesNode.java | 3 - 31 files changed, 1541 insertions(+), 790 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6b5dfbf6c3..11d556c819 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -224,7 +224,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - boolean reportDeviceActivity = false; + boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { @@ -263,7 +263,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { callback.onSuccess(); } - //TODO 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 890128e645..8adc6e8ac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -19,6 +19,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; @@ -26,7 +27,7 @@ public interface ActorService extends SessionMsgProcessor { void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - void onMsg(SendToClusterMsg msg); + void onMsg(TbActorMsg msg); void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index fc9eb537c1..cb681182f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; @@ -108,7 +109,7 @@ public class DefaultActorService implements ActorService { } @Override - public void onMsg(SendToClusterMsg msg) { + public void onMsg(TbActorMsg msg) { appActor.tell(msg, ActorRef.noSender()); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 07b7704d06..27d5453add 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -24,6 +24,7 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -34,6 +35,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -62,15 +67,19 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final DeviceStateService stateService; + private final LocalSubscriptionService localSubscriptionService; + private final SubscriptionManagerService subscriptionManagerService; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService) { + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, LocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService) { this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); this.actorContext = actorContext; this.stateService = stateService; + this.localSubscriptionService = localSubscriptionService; + this.subscriptionManagerService = subscriptionManagerService; } @PostConstruct @@ -108,8 +117,15 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } else if (toCoreMsg.hasDeviceStateServiceMsg()) { log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); } } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); callback.onFailure(e); } }); @@ -126,23 +142,10 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } } + log.info("Tb Core Consumer stopped."); }); } - private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(deviceStateServiceMsg); - } - stateService.onQueueMsg(deviceStateServiceMsg, callback); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); - } - @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { @@ -161,4 +164,55 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } + private void forwardToLocalSubMgrService(TransportProtos.LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { + if (msg.hasSubUpdate()) { + localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToSubMgrService(TransportProtos.SubscriptionMgrMsgProto msg, TbMsgCallback callback) { + if (msg.hasAttributeSub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); + } else if (msg.hasTelemetrySub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); + } else if (msg.hasSubClose()) { + TransportProtos.TbSubscriptionCloseProto closeProto = msg.getSubClose(); + subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); + } else if (msg.hasTsUpdate()) { + TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + subscriptionManagerService.onTimeseriesDataUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); + } else if (msg.hasAttrUpdate()) { + TransportProtos.TbAttributeUpdateProto proto = msg.getAttrUpdate(); + subscriptionManagerService.onAttributesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + private void throwNotHandled(Object msg, TbMsgCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 774377407a..85fec1d2ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -17,6 +17,19 @@ package org.thingsboard.server.service.queue; public interface TbMsgCallback { + TbMsgCallback EMPTY = new TbMsgCallback() { + + @Override + public void onSuccess() { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(); void onFailure(Throwable t); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 5ee26bc294..51f544712e 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -92,7 +92,6 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; */ @Service @Slf4j -//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -150,7 +149,6 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); private ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); @@ -378,7 +376,6 @@ public class DefaultDeviceStateService implements DeviceStateService { deviceStates.put(state.getDeviceId(), state); } - //TODO 2.5: review this method private void updateState() { long ts = System.currentTimeMillis(); Set deviceIds = new HashSet<>(deviceStates.keySet()); @@ -439,13 +436,9 @@ public class DefaultDeviceStateService implements DeviceStateService { deviceStates.remove(deviceId); deviceLastReportedActivity.remove(deviceId); deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } - } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + Set deviceIdSet = partitionedDevices.get(tpi); + deviceIdSet.remove(deviceId); } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java new file mode 100644 index 0000000000..f9d60b37a5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java @@ -0,0 +1,230 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class DefaultLocalSubscriptionService implements LocalSubscriptionService { + + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); + + @Autowired + private TelemetryWebSocketService wsService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbCoreQueueProvider coreQueueProvider; + + @Autowired + @Lazy + private SubscriptionManagerService subscriptionManagerService; + + private ExecutorService wsCallBackExecutor; + private TbQueueProducer> toCoreProducer; + + @PostConstruct + public void initExecutor() { + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (wsCallBackExecutor != null) { + wsCallBackExecutor.shutdownNow(); + } + } + + @Override + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + } + } + + @Override + @EventListener(ClusterTopologyChangeEvent.class) + public void onApplicationEvent(ClusterTopologyChangeEvent event) { + if (event.getServiceKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { + /* + * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. + * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. + * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically + * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. + * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache + * Since number of subscriptions is usually much less then number of devices that are pushing data. + */ + subscriptionsBySessionId.values().forEach(map -> map.values() + .forEach(sub -> pushSubscriptionToManagerService(sub, false))); + } + } + + //TODO 3.1: replace null callbacks with callbacks from websocket service. + @Override + public void addSubscription(TbSubscription subscription) { + EntityId entityId = subscription.getEntityId(); + // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { + subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); + } + pushSubscriptionToManagerService(subscription, true); + registerSubscription(subscription); + } + + private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + if (pushToLocalService) { + subscriptionManagerService.addSubscription(subscription, TbMsgCallback.EMPTY); + } + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + } + } + + @Override + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback) { + TbSubscription subscription = subscriptionsBySessionId + .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); + if (subscription != null) { + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; + update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); + break; + case ATTRIBUTES: + TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; + update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); + break; + } + wsService.sendWsMsg(sessionId, update); + } + callback.onSuccess(); + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + if (sessionSubscriptions.isEmpty()) { + subscriptionsBySessionId.remove(sessionId); + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbMsgCallback.EMPTY); + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + } + + @Override + public void cancelAllSessionSubscriptions(String sessionId) { + Map subscriptions = subscriptionsBySessionId.get(sessionId); + if (subscriptions != null) { + Set toRemove = new HashSet<>(subscriptions.keySet()); + toRemove.forEach(id -> cancelSubscription(sessionId, id)); + } + } + + private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); + + Map keyStates; + if (subscription.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = subscription.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + return TbTimeseriesSubscription.builder() + .serviceId(subscription.getServiceId()) + .sessionId(subscription.getSessionId()) + .subscriptionId(subscription.getSubscriptionId()) + .tenantId(subscription.getTenantId()) + .entityId(entityView.getEntityId()) + .startTime(entityView.getStartTimeMs()) + .endTime(entityView.getEndTimeMs()) + .allKeys(false) + .keyStates(keyStates).build(); + } + + private void registerSubscription(TbSubscription subscription) { + Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java new file mode 100644 index 0000000000..267ea14ac1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -0,0 +1,377 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.function.Predicate; + +@Slf4j +@Service +public class DefaultSubscriptionManagerService implements SubscriptionManagerService { + + @Autowired + private AttributesService attrService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + + @Autowired + private TbCoreQueueProvider coreQueueProvider; + + @Autowired + private LocalSubscriptionService localSubscriptionService; + + @Autowired + private DeviceStateService deviceStateService; + + @Autowired + private ActorService actorService; + + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + + private ExecutorService tsCallBackExecutor; + private String serviceId; + private TbQueueProducer> toCoreProducer; + + @PostConstruct + public void initExecutor() { + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + serviceId = serviceInfoProvider.getServiceId(); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (tsCallBackExecutor != null) { + tsCallBackExecutor.shutdownNow(); + } + } + + @Override + public void addSubscription(TbSubscription subscription, TbMsgCallback callback) { + log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", + subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); + } + boolean newSubscription = subscriptionsByEntityId + .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); + subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); + if (newSubscription) { + switch (subscription.getType()) { + case TIMESERIES: + handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); + break; + case ATTRIBUTES: + handleNewAttributeSubscription((TbAttributeSubscription) subscription); + break; + } + } + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + removeSubscriptionFromEntityMap(subscription); + removeSubscriptionFromPartitionMap(subscription); + if (sessionSubscriptions.isEmpty()) { + subscriptionsByWsSessionId.remove(sessionId); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } + + @Override + public void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { + return (TbTimeseriesSubscription) s; + } else { + return null; + } + }, s -> true, s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + callback.onSuccess(); + } + + @Override + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { + return (TbAttributeSubscription) s; + } else { + return null; + } + }, + s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), + s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + if (entityId.getEntityType() == EntityType.DEVICE) { + if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } + } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); + actorService.onMsg(notificationMsg); + } + } + callback.onSuccess(); + } + + private void onLocalSubUpdate(EntityId entityId, + Function castFunction, + Predicate filterFunction, + Function> processFunction) { + Set entitySubscriptions = subscriptionsByEntityId.get(entityId); + if (entitySubscriptions != null) { + entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { + List subscriptionUpdate = processFunction.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + if (serviceId.equals(s.getServiceId())) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); + } else { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); + toCoreProducer.send(tpi, toProto(s, subscriptionUpdate), null); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void removeSubscriptionFromEntityMap(TbSubscription sub) { + Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); + if (entitySubSet != null) { + entitySubSet.remove(sub); + if (entitySubSet.isEmpty()) { + subscriptionsByEntityId.remove(sub.getEntityId()); + } + } + } + + private void removeSubscriptionFromPartitionMap(TbSubscription sub) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); + Set subs = partitionedSubscriptions.get(tpi); + if (subs != null) { + subs.remove(sub); + } + } + + private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } + + private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().forEach((key, value) -> { + if (curTs > value) { + long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); + long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; + queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); + + builder.setSessionId(subscription.getSessionId()); + builder.setSubscriptionId(subscription.getSubscriptionId()); + + Map> data = new TreeMap<>(); + for (TsKvEntry tsEntry : updates) { + List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); + Object[] value = new Object[2]; + value[0] = tsEntry.getTs(); + value[1] = tsEntry.getValueAsString(); + values.add(value); + } + + data.forEach((key, value) -> { + TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); + dataBuilder.setKey(key); + value.forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + builder.addData(dataBuilder.build()); + }); + + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToLocalSubscriptionServiceMsg( + LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) + .build(); + return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java new file mode 100644 index 0000000000..a80fb0db92 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +public interface LocalSubscriptionService { + + void addSubscription(TbSubscription subscription); + + void cancelSubscription(String sessionId, int subscriptionId); + + void cancelAllSessionSubscriptions(String sessionId); + + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback); + + void onApplicationEvent(PartitionChangeEvent event); + + void onApplicationEvent(ClusterTopologyChangeEvent event); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java new file mode 100644 index 0000000000..bb236141f8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgCallback; + +import java.util.List; + +public interface SubscriptionManagerService extends ApplicationListener { + + void addSubscription(TbSubscription subscription, TbMsgCallback callback); + + void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback); + + void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); + + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java new file mode 100644 index 0000000000..83a86efefc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbAttributeSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final TbAttributeSubscriptionScope scope; + + @Builder + public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.scope = scope; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java new file mode 100644 index 0000000000..d23cc3581d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +public enum TbAttributeSubscriptionScope { + + CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java new file mode 100644 index 0000000000..22b37ff690 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@Data +@AllArgsConstructor +public abstract class TbSubscription { + + private final String serviceId; + private final String sessionId; + private final int subscriptionId; + private final TenantId tenantId; + private final EntityId entityId; + private final TbSubscriptionType type; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbSubscription that = (TbSubscription) o; + return subscriptionId == that.subscriptionId && + sessionId.equals(that.sessionId) && + tenantId.equals(that.tenantId) && + entityId.equals(that.entityId) && + type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java new file mode 100644 index 0000000000..d7a4715460 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +public enum TbSubscriptionType { + TIMESERIES, ATTRIBUTES +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java new file mode 100644 index 0000000000..d0b4d713f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java @@ -0,0 +1,247 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +public class TbSubscriptionUtils { + + public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() + .setServiceId(subscription.getServiceId()) + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()) + .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(subscription.getEntityId().getEntityType().name()) + .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); + + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; + TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(tSub.isAllKeys()); + tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + tSubProto.setStartTime(tSub.getStartTime()); + tSubProto.setEndTime(tSub.getEndTime()); + msgBuilder.setTelemetrySub(tSubProto.build()); + break; + case ATTRIBUTES: + TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; + TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(aSub.isAllKeys()) + .setScope(aSub.getScope().name()); + aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + msgBuilder.setAttributeSub(aSubProto.build()); + break; + } + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()).build(); + msgBuilder.setSubClose(closeProto); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { + TbSubscriptionProto subProto = attributeSub.getSub(); + TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); + builder.allKeys(attributeSub.getAllKeys()); + Map keyStates = new HashMap<>(); + attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.keyStates(keyStates); + return builder.build(); + } + + public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { + TbSubscriptionProto subProto = telemetrySub.getSub(); + TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.allKeys(telemetrySub.getAllKeys()); + Map keyStates = new HashMap<>(); + telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.startTime(telemetrySub.getStartTime()); + builder.endTime(telemetrySub.getEndTime()); + builder.keyStates(keyStates); + return builder.build(); + } + + public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { + TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setTsUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setAttrUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { + KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); + switch (attr.getDataType()) { + case BOOLEAN: + attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); + break; + case LONG: + attr.getLongValue().ifPresent(dataBuilder::setLongV); + break; + case DOUBLE: + attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); + break; + case JSON: + attr.getJsonValue().ifPresent(dataBuilder::setJsonV); + break; + case STRING: + attr.getStrValue().ifPresent(dataBuilder::setStringV); + break; + } + return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); + } + + public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { + return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); + } + + public static List toTsKvEntityList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); + return result; + } + + public static List toAttributeKvList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); + return result; + } + + private static KvEntry getKvEntry(KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getType().getNumber()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongV()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStringV()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); + break; + } + return entry; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java new file mode 100644 index 0000000000..0be63f7b65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbTimeseriesSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final long startTime; + @Getter private final long endTime; + + @Builder + public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, long startTime, long endTime) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 04952a3bd1..d94eb3f2b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,71 +18,44 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.Subscription; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -91,8 +64,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - @Autowired - private TelemetryWebSocketService wsService; + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); @Autowired private AttributesService attrService; @@ -101,23 +73,24 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private TimeseriesService tsService; @Autowired - private EntityViewService entityViewService; + private TbCoreQueueProvider coreQueueProvider; @Autowired - @Lazy - private DeviceStateService stateService; + private PartitionService partitionService; @Autowired - @Lazy - private ActorService actorService; - + private SubscriptionManagerService subscriptionManagerService; + + private TbQueueProducer> toCoreProducer; + private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -130,65 +103,12 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - - @Override - public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { - long startTime = 0L; - long endTime = 0L; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); - entityId = entityView.getEntityId(); - startTime = entityView.getStartTimeMs(); - endTime = entityView.getEndTimeMs(); - sub = getUpdatedSubscriptionState(entityId, sub, entityView); - } - //TODO 2.5 - Optional server = Optional.empty();//routingService.resolveById(entityId); - Subscription subscription; - if (server.isPresent()) { - ServerAddress address = server.get(); - log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); - subscription = new Subscription(sub, true, address, startTime, endTime); - tellNewSubscription(address, sessionId, subscription); - } else { - log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); - subscription = new Subscription(sub, true, null, startTime, endTime); - } - registerSubscription(sessionId, entityId, subscription); - } - - private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { - Map keyStates; - if (sub.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = sub.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); - } - @Override - public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { - cleanupLocalWsSessionSubscriptions(sessionId); - } - - @Override - public void removeSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - Subscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); } } @@ -201,14 +121,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); + addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); } @Override @@ -235,355 +155,23 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - @Override - public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } - - @Override - public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionProto proto; - try { - proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - Map statesMap = proto.getKeyStatesList().stream().collect( - Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); - Subscription subscription = new Subscription( - new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), - new TenantId(UUID.fromString(proto.getTenantId())), - EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), - false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); - - addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); - } - - @Override - public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionUpdateProto proto; - try { - proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - SubscriptionUpdate update = convert(proto); - String sessionId = proto.getSessionId(); - log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); - Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); - if (subOpt.isPresent()) { - updateSubscriptionState(sessionId, subOpt.get(), update); - wsService.sendWsMsg(sessionId, update); - } - } - - @Override - public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionCloseProto proto; - try { - proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); - } - - @Override - public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SessionCloseProto proto; - try { - proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); - } - - @Override - public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.AttributeUpdateProto proto; - try { - proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), - proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); - } - - @Override - public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.TimeseriesUpdateProto proto; - try { - proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); - } - - @Override - public void onClusterUpdate() { - log.trace("Processing cluster onUpdate msg!"); - Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); - while (deviceIterator.hasNext()) { - Map.Entry> e = deviceIterator.next(); - Set subscriptions = e.getValue(); - //TODO 2.5 - Optional newAddressOptional = Optional.empty();// routingService.resolveById(e.getKey()); - if (newAddressOptional.isPresent()) { - newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); - } else { - checkSubscriptionsPrevAddress(subscriptions); - } - if (subscriptions.size() == 0) { - log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); - deviceIterator.remove(); - } - } - } - - private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { - Iterator subscriptionIterator = subscriptions.iterator(); - while (subscriptionIterator.hasNext()) { - Subscription s = subscriptionIterator.next(); - if (s.isLocal()) { - if (!newAddress.equals(s.getServer())) { - log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); - s.setServer(newAddress); - tellNewSubscription(newAddress, s.getWsSessionId(), s); - } - } else { - log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); - subscriptionIterator.remove(); - //TODO: onUpdate state of subscription by WsSessionId and other maps. - } - } - } - - private void checkSubscriptionsPrevAddress(Set subscriptions) { - for (Subscription s : subscriptions) { - if (s.isLocal() && s.getServer() != null) { - log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); - s.setServer(null); - } else { - log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); - } - } - } - - private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); - registerSubscription(sessionId, entityId, subscription); - if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().entrySet().forEach(e -> { - if (curTs > e.getValue()) { - queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); - } else { - log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - } - - private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - //TODO 2.5 - Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalAttributesUpdate(entityId, scope, attributes); - if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } - } + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); } else { - tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); } } - private void onTimeseriesUpdate(EntityId entityId, List ts) { - //TODO 2.5 - Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalTimeseriesUpdate(entityId, ts); + private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + subscriptionManagerService.onTimeseriesDataUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); } else { - tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); - } - } - - private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - } - - private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - } - - private boolean isInTimeRange(Subscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - deviceSubscriptions.stream().filter(filter).forEach(s -> { - String sessionId = s.getWsSessionId(); - List subscriptionUpdate = f.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - if (s.isLocal()) { - updateSubscriptionState(sessionId, s, update); - wsService.sendWsMsg(sessionId, update); - } else { - tellRemoteSubUpdate(s.getServer(), sessionId, update); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { - log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); - update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); - } - - private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { - Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); - deviceSubscriptions.add(subscription); - Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - - private void cleanupLocalWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, true); - } - - private void cleanupRemoteWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, false); - } - - private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { - log.debug("[{}] Removing all subscriptions for particular session.", sessionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - int sessionSubscriptionSize = sessionSubscriptions.size(); - - for (Subscription subscription : sessionSubscriptions.values()) { - EntityId entityId = subscription.getEntityId(); - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - deviceSubscriptions.remove(subscription); - if (deviceSubscriptions.isEmpty()) { - subscriptionsByEntityId.remove(entityId); - } - } - subscriptionsByWsSessionId.remove(sessionId); - log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); - - if (localSession) { - notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); - } - } else { - log.debug("[{}] No subscriptions found!", sessionId); - } - } - - private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { - Set affectedServers = new HashSet<>(); - for (Subscription subscription : sessionSubscriptions.values()) { - if (subscription.getServer() != null) { - affectedServers.add(subscription.getServer()); - } - } - for (ServerAddress address : affectedServers) { - log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); - tellRemoteSessionClose(address, sessionId); - } - } - - private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - if (subscription.isLocal() && subscription.getServer() != null) { - tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); - } - if (sessionSubscriptions.isEmpty()) { - log.debug("[{}] Removed last subscription for particular session.", sessionId); - subscriptionsByWsSessionId.remove(sessionId); - } else { - log.debug("[{}] Removed session subscription.", sessionId); - } - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - boolean result = deviceSubscriptions.remove(subscription); - if (result) { - if (deviceSubscriptions.size() == 0) { - log.debug("[{}] Removed last subscription for particular device.", sessionId); - subscriptionsByEntityId.remove(entityId); - } else { - log.debug("[{}] Removed device subscription.", sessionId); - } - } else { - log.debug("[{}] Subscription not found!", sessionId); - } - } else { - log.debug("[{}] No device subscriptions found!", sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); } } @@ -613,167 +201,4 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } - - private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { - ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(sub.getSubscriptionId()); - builder.setTenantId(sub.getSub().getTenantId().getId().toString()); - builder.setEntityType(sub.getEntityId().getEntityType().name()); - builder.setEntityId(sub.getEntityId().getId().toString()); - builder.setType(sub.getType().name()); - builder.setAllKeys(sub.isAllKeys()); - if (sub.getScope() != null) { - builder.setScope(sub.getScope()); - } - sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( - ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { - ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(update.getSubscriptionId()); - builder.setErrorCode(update.getErrorCode()); - if (update.getErrorMsg() != null) { - builder.setErrorMsg(update.getErrorMsg()); - } - update.getData().entrySet().forEach( - e -> { - ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); - - dataBuilder.setKey(e.getKey()); - e.getValue().forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - - builder.addData(dataBuilder.build()); - } - ); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { - ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { - ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSessionClose(ServerAddress address, String sessionId) { - ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { - ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { - ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setTs(ts); - dataBuilder.setValueType(attr.getDataType().ordinal()); - switch (attr.getDataType()) { - case BOOLEAN: - Optional booleanValue = attr.getBooleanValue(); - booleanValue.ifPresent(dataBuilder::setBoolValue); - break; - case LONG: - Optional longValue = attr.getLongValue(); - longValue.ifPresent(dataBuilder::setLongValue); - break; - case DOUBLE: - Optional doubleValue = attr.getDoubleValue(); - doubleValue.ifPresent(dataBuilder::setDoubleValue); - break; - case JSON: - Optional jsonValue = attr.getJsonValue(); - jsonValue.ifPresent(dataBuilder::setJsonValue); - break; - case STRING: - Optional stringValue = attr.getStrValue(); - stringValue.ifPresent(dataBuilder::setStrValue); - break; - } - return dataBuilder; - } - - private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { - return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); - } - - private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { - return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); - } - - private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getValueType()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); - break; - } - return entry; - } - - private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - private Optional getSubscription(String sessionId, int subscriptionId) { - Subscription state = null; - Map subMap = subscriptionsByWsSessionId.get(sessionId); - if (subMap != null) { - state = subMap.get(subscriptionId); - } - return Optional.ofNullable(state); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index c4fe716ad9..51556602f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -42,24 +42,25 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; +import org.thingsboard.server.service.subscription.TbAttributeSubscription; +import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmd; import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; -import org.thingsboard.server.service.telemetry.exception.AccessDeniedException; -import org.thingsboard.server.service.telemetry.exception.EntityNotFoundException; -import org.thingsboard.server.service.telemetry.exception.InternalErrorException; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -70,7 +71,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -98,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TelemetrySubscriptionService subscriptionManager; + private LocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -112,6 +112,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -127,9 +130,11 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; + private String serviceId; @PostConstruct public void initExecutor() { + serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -153,7 +158,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); processSessionClose(sessionRef); break; } @@ -334,8 +339,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -421,8 +434,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -494,8 +515,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -520,12 +549,19 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); - Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -544,9 +580,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); } else { - subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); + subService.cancelSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d77a0e50dc..e905559516 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,34 +15,17 @@ */ package org.thingsboard.server.service.telemetry; +import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { - void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); - - void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); - - void removeSubscription(String sessionId, int cmdId); - - void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); - - void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); - - void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 68aaca34c2..992dc26af9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { + return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto index 46526e8e1c..f0ca4d6d7c 100644 --- a/application/src/main/proto/cluster.proto +++ b/application/src/main/proto/cluster.proto @@ -64,69 +64,7 @@ enum MessageType { } // Messages related to CLUSTER_TELEMETRY_MESSAGE -message SubscriptionProto { - string sessionId = 1; - int32 subscriptionId = 2; - string entityType = 3; - string tenantId = 4; - string entityId = 5; - string type = 6; - bool allKeys = 7; - repeated SubscriptionKetStateProto keyStates = 8; - string scope = 9; -} - -message SubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated SubscriptionUpdateValueListProto data = 5; -} - -message AttributeUpdateProto { - string entityType = 1; - string entityId = 2; - string scope = 3; - repeated KeyValueProto data = 4; -} - -message TimeseriesUpdateProto { - string entityType = 1; - string entityId = 2; - repeated KeyValueProto data = 4; -} - -message SessionCloseProto { - string sessionId = 1; -} - -message SubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message SubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message SubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} -message KeyValueProto { - string key = 1; - int64 ts = 2; - int32 valueType = 3; - string strValue = 4; - int64 longValue = 5; - double doubleValue = 6; - bool boolValue = 7; - string jsonValue = 8; -} message FromDeviceRPCResponseProto { int64 requestIdMSB = 1; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fc49716d25..8a4d735d26 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -412,7 +412,7 @@ audit-log: state: defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" - persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:true}" js: evaluator: "${JS_EVALUATOR:local}" # local/remote diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 5639f98d01..2993d81154 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,6 +32,10 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } + public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { + this(kv, lastUpdateTs); + } + @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java new file mode 100644 index 0000000000..fb379c213a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.Set; + + +public class ClusterTopologyChangeEvent extends ApplicationEvent { + + @Getter + private final Set serviceKeys; + + public ClusterTopologyChangeEvent(Object source, Set serviceKeys) { + super(source); + this.serviceKeys = serviceKeys; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 97b660a48c..f5efec30d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -66,6 +66,10 @@ public class ConsistentHashPartitionService implements PartitionService { //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + private Map tbCoreNotificationTopics = new HashMap<>(); + private Map tbRuleEngineNotificationTopics = new HashMap<>(); + private List currentOtherServices; + private HashFunction hashFunction; public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { @@ -85,12 +89,11 @@ public class ConsistentHashPartitionService implements PartitionService { @Override public List getCurrentPartitions(ServiceType serviceType) { ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); - TenantId tenantId = getTenantId(currentService); + TenantId tenantId = getSystemOrIsolatedTenantId(currentService); ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); List partitions = myPartitions.get(serviceKey); List topicPartitions = new ArrayList<>(); for (Integer partition : partitions) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -112,34 +115,16 @@ public class ConsistentHashPartitionService implements PartitionService { return buildTopicPartitionInfo(serviceType, tenantId, partition); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { - return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); - tpi.partition(partition); - if (isolated) { - tpi.tenantId(tenantId); - } - return tpi.build(); - } - @Override public void recalculatePartitions(ServiceInfo currentService, List otherServices) { logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); - Map> newCircles = new HashMap<>(ServiceType.values().length); - for (ServiceType serverType : ServiceType.values()) { - newCircles.put(serverType, new ConsistentHashCircle<>()); - } - addNode(newCircles, currentService); + Map> circles = new HashMap<>(); + addNode(circles, currentService); for (ServiceInfo other : otherServices) { - addNode(newCircles, other); - TenantId tenantId = getTenantId(other); + TenantId tenantId = getSystemOrIsolatedTenantId(other); + addNode(circles, other); if (!tenantId.isNullUid()) { isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); for (String serviceType : other.getServiceTypesList()) { @@ -149,12 +134,14 @@ public class ConsistentHashPartitionService implements PartitionService { } } ConcurrentMap> oldPartitions = myPartitions; + TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { + ServiceKey myServiceKey = new ServiceKey(type, myTenantId); for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceKey), i); if (currentService.equals(serviceInfo)) { - ServiceKey serviceKey = new ServiceKey(type, getTenantId(serviceInfo)); + ServiceKey serviceKey = new ServiceKey(type, getSystemOrIsolatedTenantId(serviceInfo)); myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); } } @@ -165,13 +152,81 @@ public class ConsistentHashPartitionService implements PartitionService { Set tpiList = partitions.stream() .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) .collect(Collectors.toSet()); + // Adding notifications topic for every @TopicPartitionInfo list + tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } + }); + + if (currentOtherServices == null) { + currentOtherServices = new ArrayList<>(otherServices); + } else { + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); + currentOtherServices = otherServices; + currentMap.forEach((key, list) -> { + if (!list.equals(newMap.get(key))) { + changes.add(key); + + } + }); + currentMap.keySet().forEach(newMap::remove); + changes.addAll(newMap.keySet()); + if (!changes.isEmpty()) { + applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + } + } + } + + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + } + }); + return currentMap; + } + + @Override + public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { + switch (serviceType) { + case TB_CORE: + return tbCoreNotificationTopics.computeIfAbsent(serviceId, + id -> buildTopicPartitionInfo(serviceType, serviceId)); + case TB_RULE_ENGINE: + return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, + id -> buildTopicPartitionInfo(serviceType, serviceId)); + default: + return buildTopicPartitionInfo(serviceType, serviceId); + } + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { + return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (isolated) { + tpi.tenantId(tenantId); + } + return tpi.build(); } private void logServiceInfo(TransportProtos.ServiceInfo server) { - TenantId tenantId = getTenantId(server); + TenantId tenantId = getSystemOrIsolatedTenantId(server); if (tenantId.isNullUid()) { log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); } else { @@ -179,21 +234,23 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private TenantId getTenantId(TransportProtos.ServiceInfo serviceInfo) { + private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); } - private void addNode(Map> circles, ServiceInfo instance) { + private void addNode(Map> circles, ServiceInfo instance) { + TenantId tenantId = getSystemOrIsolatedTenantId(instance); for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); for (int i = 0; i < virtualNodesSize; i++) { - circles.get(serviceType).put(hash(instance, i).asLong(), instance); + circles.computeIfAbsent(serviceKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); } } } private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { - if (circle.isEmpty()) { + if (circle == null || circle.isEmpty()) { return null; } Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index f8613f5fb4..46864af79f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -36,4 +36,13 @@ public interface PartitionService { * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + + /** + * Each Service should start a consumer for messages that target individual service instance based on serviceId. + * This topic is likely to have single partition, and is always assigned to the service. + * @param tbCore + * @param serviceId + * @return + */ + TopicPartitionInfo getNotificationsTopic(ServiceType tbCore, String serviceId); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 09c29aa5e1..52086b3058 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 17f8b9c31c..19e8d64e87 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -239,6 +239,79 @@ message DeviceActorToTransportMsg { ToServerRpcResponseMsg toServerResponse = 7; } +/** + * TB Core Data Structures + */ + +message TbSubscriptionProto { + string serviceId = 1; + string sessionId = 2; + int32 subscriptionId = 3; + string entityType = 4; + int64 tenantIdMSB = 5; + int64 tenantIdLSB = 6; + int64 entityIdMSB = 7; + int64 entityIdLSB = 8; +} + +message TbTimeSeriesSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + int64 startTime = 4; + int64 endTime = 5; +} + +message TbAttributeSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + string scope = 4; +} + +message TbSubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated TbSubscriptionUpdateValueListProto data = 5; +} + +message TbAttributeUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + string scope = 6; + repeated TsKvProto data = 7; +} + +message TbTimeSeriesUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + repeated TsKvProto data = 6; +} + +message TbSubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message TbSubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message TbSubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + /** * TB Core to TB Core messages */ @@ -253,27 +326,41 @@ message DeviceStateServiceMsgProto { bool deleted = 7; } +message SubscriptionMgrMsgProto { + TbTimeSeriesSubscriptionProto telemetrySub = 1; + TbAttributeSubscriptionProto attributeSub = 2; + TbSubscriptionCloseProto subClose = 3; + TbTimeSeriesUpdateProto tsUpdate = 4; + TbAttributeUpdateProto attrUpdate = 5; +} + +message LocalSubscriptionServiceMsgProto { + TbSubscriptionUpdateProto subUpdate = 1; +} + /** * Main messages; */ /* Request from Transport Service to ThingsBoard Core Service */ message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; } /* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; } /* Messages that are handled by ThingsBoard Core Service */ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; DeviceStateServiceMsgProto deviceStateServiceMsg = 2; + SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 4; } /* Messages that are handled by ThingsBoard RuleEngine Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index fcb630d041..d8d827aa52 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -136,6 +136,7 @@ public class DefaultTransportService implements TransportService { log.warn("Failed to process the notification.", e); } }); + transportNotificationsConsumer.commit(); } catch (Exception e) { log.warn("Failed to obtain messages from queue.", e); try { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index f57849fa35..d2e19652a2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,6 +44,4 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); - void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); - } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 46a68d3e26..4ddd082f3e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -65,9 +65,6 @@ public class TbMsgAttributesNode implements TbNode { String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); - if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { - ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); - } } @Override From 99d322d4497f9d37d3fd1ff27578c6edeb948913 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 10:35:54 +0200 Subject: [PATCH 020/102] Refactoring of the rule engine --- .../server/actors/app/AppActor.java | 18 +---- .../device/DeviceActorMessageProcessor.java | 24 +----- .../actors/ruleChain/DefaultTbContext.java | 26 ++++--- .../actors/ruleChain/RuleChainActor.java | 7 +- .../RuleChainActorMessageProcessor.java | 13 ++-- .../actors/shared/EntityActorsManager.java | 6 ++ .../server/actors/tenant/TenantActor.java | 35 +++++---- .../server/controller/BaseController.java | 6 +- .../controller/RuleChainController.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 24 +++--- .../server/service/queue/MsgPackCallback.java | 1 + .../service/rpc/DefaultDeviceRpcService.java | 6 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../state/DefaultDeviceStateService.java | 8 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultLocalSubscriptionService.java | 2 +- .../DefaultSubscriptionManagerService.java | 3 +- .../LocalSubscriptionService.java | 2 +- .../SubscriptionManagerService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../BaseRuleChainTransactionService.java | 2 +- .../msg/TransportToDeviceActorMsgWrapper.java | 2 +- ...AbstractRuleEngineFlowIntegrationTest.java | 21 ++--- ...actRuleEngineLifecycleIntegrationTest.java | 10 +-- .../script/RuleNodeJsScriptEngineTest.java | 15 ++-- .../server/common/msg/MsgType.java | 8 +- .../thingsboard/server/common/msg/TbMsg.java | 49 ++++-------- .../QueueToRuleEngineMsg.java} | 6 +- .../common/msg}/queue/TbMsgCallback.java | 2 +- common/queue/src/main/proto/queue.proto | 5 +- .../service/DefaultTransportService.java | 77 ++++++++++++++++--- .../common/transport/util}/JsonUtils.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 3 +- .../engine/rest/TbRedisQueueProcessor.java | 3 +- .../TbSynchronizationBeginNode.java | 3 +- .../rule/engine/action/TbAlarmNodeTest.java | 12 +-- .../engine/filter/TbJsFilterNodeTest.java | 7 +- .../engine/filter/TbJsSwitchNodeTest.java | 3 +- .../engine/mail/TbMsgToEmailNodeTest.java | 3 +- .../TbGetCustomerAttributeNodeTest.java | 17 ++-- .../transform/TbChangeOriginatorNodeTest.java | 7 +- .../transform/TbTransformMsgNodeTest.java | 7 +- 43 files changed, 249 insertions(+), 209 deletions(-) rename common/message/src/main/java/org/thingsboard/server/common/msg/{system/ServiceToRuleEngineMsg.java => queue/QueueToRuleEngineMsg.java} (85%) rename {application/src/main/java/org/thingsboard/server/service => common/message/src/main/java/org/thingsboard/server/common/msg}/queue/TbMsgCallback.java (94%) rename {application/src/main/java/org/thingsboard/server/utils => common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util}/JsonUtils.java (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 06f936a9e6..9d80d63b2a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,14 +20,9 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; -import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; @@ -43,17 +38,12 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - public class AppActor extends RuleChainManagerActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); @@ -97,8 +87,8 @@ public class AppActor extends RuleChainManagerActor { case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -145,7 +135,7 @@ public class AppActor extends RuleChainManagerActor { // } } - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { // this may be a notification about system entities created. // log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 11d556c819..1844539b2c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -65,12 +65,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.utils.JsonUtils; +import org.thingsboard.server.common.transport.util.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -224,6 +224,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { + //TODO 2.5 boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); @@ -329,23 +330,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { - JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), - TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - - private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { - for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsgMetaData metaData = defaultMetaData.copy(); - metaData.putValue("ts", tsKv.getTs() + ""); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - } - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { UUID sessionId = getSessionId(sessionInfo); JsonObject json = new JsonObject(); @@ -354,7 +338,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { TbMsgMetaData requestMetaData = defaultMetaData.copy(); requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 2de491c89d..337a234e2d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; @@ -47,12 +46,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -71,7 +69,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -146,23 +143,25 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), 0); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(),null); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), 0); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), null); } @Override public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); + mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new QueueToRuleEngineMsg(getTenantId(), msg))); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), + getActionMetaData(ruleNodeId), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process customer created msg: " + e); } @@ -171,7 +170,8 @@ class DefaultTbContext implements TbContext { public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process device created msg: " + e); } @@ -180,7 +180,8 @@ class DefaultTbContext implements TbContext { public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process asset created msg: " + e); } @@ -189,7 +190,8 @@ class DefaultTbContext implements TbContext { public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process alarm created msg: " + e); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 72faaef533..76f831be98 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.actors.ruleChain; -import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; @@ -26,7 +25,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { @@ -43,8 +42,8 @@ public class RuleChainActor extends ComponentActor> msgs = consumer.poll(pollDuration); - if(msgs.isEmpty()){ + if (msgs.isEmpty()) { continue; } ConcurrentMap> ackMap = msgs.stream().collect( @@ -96,9 +101,10 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToRuleEngineMsg()) { - forwardToRuleEngineActor(toRuleEngineMsg.getToRuleEngineMsg(), callback); + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); } else { callback.onSuccess(); } @@ -122,16 +128,16 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } - //TODO 2.5 - private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - log.info("Received RULE ENGINE msg: {}", toRuleEngineMsg); + private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); + log.info("[{}] Received RULE ENGINE msg: {}", tbMsg.getType(), tbMsg); + actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } -// actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); } - @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${queue.rule_engine.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { stats.printStats(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d275adc4be..d616bbef16 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java index 8baecbf37a..d8f12b8f0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -39,7 +39,7 @@ import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; @@ -185,8 +185,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); + , null, null, null); + actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new QueueToRuleEngineMsg(msg.getTenantId(), tbMsg))); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 73bdd27b78..1454b0c6fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -94,7 +94,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, msg.getDataType(), newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getCallback()); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 51f544712e..816479b1e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; @@ -61,7 +61,7 @@ import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -501,8 +501,8 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + , null, null, null); + actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new QueueToRuleEngineMsg(stateData.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 14a25c2e9d..430dd77a36 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; /** * Created by ashvayka on 01.05.18. diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java index f9d60b37a5..89f745260a 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 267ea14ac1..be186e0ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -49,7 +48,7 @@ import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java index a80fb0db92..a71155f0ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java @@ -17,7 +17,7 @@ package org.thingsboard.server.service.subscription; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; public interface LocalSubscriptionService { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index bb236141f8..d4d9fa5841 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d94eb3f2b2..58ed13c1e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 173c293363..b9424d49d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -113,7 +113,7 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ @Override public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data), msg -> { + endLocalTransaction(TbMsg.fromBytes(data, null), msg -> { }, error -> { }); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 880d7507ce..16e0dd67f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; import java.util.UUID; diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 6bb1f4d99d..265a552efb 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -16,9 +16,6 @@ package org.thingsboard.server.rules.flow; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; @@ -30,25 +27,22 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.rule.RuleChainService; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -154,9 +148,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), "CUSTOM", device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + new TbMsgMetaData(), TbMsgDataType.JSON, + "{}", null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); @@ -270,8 +264,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule "CUSTOM", device.getId(), new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + TbMsgDataType.JSON, + "{}", null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 5f806d1ea0..a4360d51e9 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.rules.lifecycle; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; @@ -38,13 +37,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -141,9 +140,10 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac "CUSTOM", device.getId(), new TbMsgMetaData(), + TbMsgDataType.JSON, "{}", - null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 82b3df9736..3c5f86a3fc 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,6 +24,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -78,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -94,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -112,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -126,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData,TbMsgDataType.JSON, rawJson, null, null, null); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -147,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -169,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 04b0cae56b..4076552863 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; + /** * Created by ashvayka on 15.03.18. */ @@ -43,11 +45,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages from the REST API/SERVICE layer to the new rule engine. + * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. * - * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} + * See {@link QueueToRuleEngineMsg} */ - SERVICE_TO_RULE_ENGINE_MSG, + QUEUE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 4e7090f934..8dc16a6776 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -17,12 +17,14 @@ package org.thingsboard.server.common.msg; import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; import java.nio.ByteBuffer; @@ -32,6 +34,7 @@ import java.util.UUID; * Created by ashvayka on 13.01.18. */ @Data +@Builder @AllArgsConstructor public final class TbMsg implements Serializable { @@ -42,29 +45,14 @@ public final class TbMsg implements Serializable { private final TbMsgDataType dataType; private final String data; private final TbMsgTransactionData transactionData; - - //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - private final long clusterPartition; - - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this.id = id; - this.type = type; - this.originator = originator; - this.metaData = metaData; - this.data = data; - this.dataType = TbMsgDataType.JSON; - this.transactionData = new TbMsgTransactionData(id, originator); - this.ruleChainId = ruleChainId; - this.ruleNodeId = ruleNodeId; - this.clusterPartition = clusterPartition; - } + //This field is not serialized because we use queues and there is no need to do it + private final TbMsgCallback callback; public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); } public static byte[] toByteArray(TbMsg msg) { @@ -105,36 +93,31 @@ public final class TbMsg implements Serializable { } - public static ByteBuffer toBytes(TbMsg msg) { - return ByteBuffer.wrap(toByteArray(msg)); - } - - public static TbMsg fromBytes(byte[] data) { - return fromBytes(ByteBuffer.wrap(data)); - } - - public static TbMsg fromBytes(ByteBuffer buffer) { + public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; + if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { + ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); + public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, callback); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java similarity index 85% rename from common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index a717af85a4..6f93301a48 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.system; +package org.thingsboard.server.common.msg.queue; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -27,13 +27,13 @@ import java.io.Serializable; * Created by ashvayka on 15.03.18. */ @Data -public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { +public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; @Override public MsgType getMsgType() { - return MsgType.SERVICE_TO_RULE_ENGINE_MSG; + return MsgType.QUEUE_TO_RULE_ENGINE_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 85fec1d2ab..4f0e383fe9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.msg.queue; public interface TbMsgCallback { diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 19e8d64e87..18c47e542b 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -226,7 +226,6 @@ message TransportToRuleEngineMsg { PostAttributeMsg postAttributes = 3; ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; ToServerRpcRequestMsg toServerRPCCallRequest = 5; - SubscriptionInfoProto subscriptionInfo = 6; } message DeviceActorToTransportMsg { @@ -365,7 +364,9 @@ message ToCoreMsg { /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { - TransportToRuleEngineMsg toRuleEngineMsg = 1; + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + bytes tbMsg = 3; } /* Messages that are handled by ThingsBoard Transport Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index d8d827aa52..1a80aa4a6e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -15,11 +15,20 @@ */ package org.thingsboard.server.common.transport.service; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsgMetadata; @@ -50,16 +59,19 @@ import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by ashvayka on 17.10.18. @@ -82,6 +94,7 @@ public class DefaultTransportService implements TransportService { @Value("${queue.transport.poll_interval}") private int notificationsPollDuration; + private final Gson gson = new Gson(); private final TransportQueueProvider queueProvider; private final PartitionService partitionService; @@ -221,8 +234,19 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setPostTelemetry(msg).build(), callback); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("ts", tsKv.getTs() + ""); + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), + deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); + sendToRuleEngine(tenantId, tbMsg, packCallback); + } } } @@ -230,8 +254,15 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setPostAttributes(msg).build(), callback); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, + TbMsgDataType.JSON, gson.toJson(json), null, null, null); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } @@ -278,8 +309,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setToServerRPCCallRequest(msg).build(), callback); +// sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). +// setToServerRPCCallRequest(msg).build(), new TransportTbQueueCallback(callback)); } } @@ -455,12 +486,12 @@ public class DefaultTransportService implements TransportService { new TransportTbQueueCallback(callback) : null); } - protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); - ruleEngineMsgProducer.send(tpi, - new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(tbMsg))) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); } private class TransportTbQueueCallback implements TbQueueCallback { @@ -480,4 +511,26 @@ public class DefaultTransportService implements TransportService { DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); } } + + private class MsgPackCallback implements TbQueueCallback { + private final AtomicInteger msgCount; + private final TransportServiceCallback callback; + + public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { + this.msgCount = new AtomicInteger(msgCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (msgCount.decrementAndGet() <= 0) { + callback.onSuccess(null); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onError(t); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonUtils.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java index 5621362c09..583b73dba1 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.transport.util; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 2e99e75b53..1952985e53 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -75,7 +75,8 @@ public class TbMsgCountNode implements TbNode { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); + //TODO 2.5: Callback? + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, null); ctx.tellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java index cff9faaab5..3349706221 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -66,7 +66,8 @@ class TbRedisQueueProcessor { if (listOperations.size(redisKey) > 0) { List list = listOperations.range(redisKey, -10, -1); list.forEach(obj -> { - TbMsg msg = TbMsg.fromBytes((byte[]) obj); + //TODO 2.5: Callback? + TbMsg msg = TbMsg.fromBytes((byte[]) obj, null); log.debug("Trying to send the message: {}", msg); listOperations.remove(redisKey, -1, obj); httpClient.processMessage(ctx, msg, this); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 5423236499..a6bd7b6164 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -57,8 +57,9 @@ public class TbSynchronizationBeginNode implements TbNode { log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); + //TODO 2.5: Callback? TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), null); ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 67c5b1cc18..e0b76f1cda 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -37,7 +37,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -99,7 +101,7 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -141,7 +143,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -164,7 +166,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -209,7 +211,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -256,7 +258,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index 27feed93ba..c840e708eb 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -58,7 +59,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -71,7 +72,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -84,7 +85,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 1e752b55b8..4934e14062 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -65,7 +66,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index ace02bacd7..8315d8e40b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; @@ -62,7 +63,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0af1cef8ae..0169cadcb2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.*; import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -102,7 +103,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -127,7 +128,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -152,7 +153,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +167,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); entityAttributeFetched(customerId); } @@ -177,7 +178,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -192,7 +193,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -207,7 +208,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -234,7 +235,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 4c1edce8f6..1ed66fc786 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -91,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -119,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -146,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index aa9ad8b76f..9a315f893b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -31,6 +31,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,8 +63,8 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId, null); + TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); @@ -84,7 +85,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); From 2ccce3b6d9c3d65382d2d3db7bae66861bbaad9a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 14:08:21 +0200 Subject: [PATCH 021/102] Main Logic of RuleChainActor to handle queue messages --- .../server/actors/ActorSystemContext.java | 10 + .../device/DeviceActorMessageProcessor.java | 28 +-- .../actors/ruleChain/DefaultTbContext.java | 8 +- .../actors/ruleChain/RuleChainActor.java | 6 +- .../RuleChainActorMessageProcessor.java | 214 +++++++++--------- .../ruleChain/RuleChainToRuleChainMsg.java | 1 - .../server/actors/tenant/TenantActor.java | 12 - .../DefaultTbRuleEngineConsumerService.java | 3 +- .../server/common/msg/MsgType.java | 1 - .../thingsboard/server/common/msg/TbMsg.java | 30 ++- .../MultipleTbQueueTbMsgCallbackWrapper.java | 43 ++++ .../queue/TbQueueTbMsgCallbackWrapper.java | 32 +-- .../ConsistentHashPartitionService.java | 31 ++- .../queue/discovery/PartitionService.java | 3 +- .../queue/discovery/TopicPartitionInfo.java | 12 +- .../discovery/TopicPartitionInfoKey.java | 43 ++++ .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- 17 files changed, 292 insertions(+), 187 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java rename application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java (52%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 46e6df60ab..e5158377cd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -63,7 +63,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -151,6 +153,14 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; + @Autowired + @Getter + private PartitionService partitionService; + + @Autowired + @Getter + private TbRuleEngineQueueProvider ruleEngineQueueProvider; + @Autowired @Getter private TimeseriesService tsService; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 1844539b2c..4713662067 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -331,18 +331,18 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { - UUID sessionId = getSessionId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", request.getMethodName()); - json.add("params", JsonUtils.parse(request.getParams())); - - TbMsgMetaData requestMetaData = defaultMetaData.copy(); - requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - - scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); - toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); +// UUID sessionId = getSessionId(sessionInfo); +// JsonObject json = new JsonObject(); +// json.addProperty("method", request.getMethodName()); +// json.add("params", JsonUtils.parse(request.getParams())); +// +// TbMsgMetaData requestMetaData = defaultMetaData.copy(); +// requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); +// TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); +// context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); +// +// scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); +// toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); } private TransportProtos.SessionType getSessionType(UUID sessionId) { @@ -372,10 +372,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - } - void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { if (attributeSubscriptions.size() > 0) { boolean hasNotificationData = false; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 337a234e2d..a5b5254174 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -118,10 +119,7 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - //TODO 2.5 -// Optional address = mainCtx.getRoutingService().resolveById(entityId); -// return !address.isPresent(); - return true; + return mainCtx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -143,7 +141,7 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(),null); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), null); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 76f831be98..83cde28a45 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; @@ -43,10 +42,7 @@ public class RuleChainActor extends ComponentActor { - private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; + private final PartitionService partitionService; + private final TbQueueProducer> producer; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - private String ruleChainName; RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { @@ -76,7 +83,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.ruleChainName = ruleChainId.toString(); + this.partitionService = systemContext.getPartitionService(); + this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); } @Override @@ -89,7 +97,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -110,7 +117,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); for (RuleNode ruleNode : ruleNodeList) { @@ -189,101 +195,113 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); -// if (address.isPresent()) { -// onRemoteTellNext(address.get(), envelope); -// } else { - onLocalTellNext(envelope); -// } + try { + checkActive(); + EntityId entityId = msg.getOriginator(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + RuleNodeId originatorNodeId = envelope.getOriginator(); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + //TODO 2.5: Maybe let's check that the output relation is not a Failure? + if (envelope.getRelationTypes().contains(TbRelationTypes.FAILURE)) { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, envelope.getOriginator().getId()); + msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + envelope.getOriginator().getId().toString() + "]")); + } else { + msg.getCallback().onSuccess(); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(tpi, msg, relation.getOut(), relation.getType()); + } + } else { + MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + putToQueue(tpi, msg, callbackWrapper, target); + } + } + } catch (Exception e) { + msg.getCallback().onFailure(e); + } } - private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); - envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - //TODO 2.5 -// systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { + switch (target.getEntityType()) { + case RULE_NODE: + putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); + break; + case RULE_CHAIN: + putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); + break; + } } - private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - RuleNodeId originatorNodeId = envelope.getOriginator(); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(msg, relation.getOut(), relation.getType()); + private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { + if (tpi.isMyPartition()) { + switch (target.getEntityType()) { + case RULE_NODE: + pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); + break; + case RULE_CHAIN: + parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); + break; } } else { - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - switch (target.getEntityType()) { - case RULE_NODE: - enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); - break; - case RULE_CHAIN: - enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); - break; - } - } - //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } + putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); } } + private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { + ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(newMsg))) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(newMsg.getId(), toQueueMsg), callbackWrapper); + } + private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -296,38 +314,10 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor> myPartitions = new ConcurrentHashMap<>(); //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); private Map tbCoreNotificationTopics = new HashMap<>(); private Map tbRuleEngineNotificationTopics = new HashMap<>(); @@ -87,12 +89,12 @@ public class ConsistentHashPartitionService implements PartitionService { } @Override - public List getCurrentPartitions(ServiceType serviceType) { + public Set getCurrentPartitions(ServiceType serviceType) { ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); TenantId tenantId = getSystemOrIsolatedTenantId(currentService); ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); List partitions = myPartitions.get(serviceKey); - List topicPartitions = new ArrayList<>(); + Set topicPartitions = new LinkedHashSet<>(); for (Integer partition : partitions) { TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); @@ -112,7 +114,9 @@ public class ConsistentHashPartitionService implements PartitionService { .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); int partition = Math.abs(hash % partitionSizes.get(serviceType)); - return buildTopicPartitionInfo(serviceType, tenantId, partition); + boolean isolatedTenant = isIsolated(serviceType, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceType, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceType, tenantId, partition)); } @Override @@ -156,8 +160,8 @@ public class ConsistentHashPartitionService implements PartitionService { tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } - }); + tpiCache.clear(); if (currentOtherServices == null) { currentOtherServices = new ArrayList<>(otherServices); @@ -207,7 +211,7 @@ public class ConsistentHashPartitionService implements PartitionService { } private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null); + return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null, false); } private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { @@ -215,16 +219,29 @@ public class ConsistentHashPartitionService implements PartitionService { } private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); - if (isolated) { + ServiceKey myPartitionsSearchKey; + if (isIsolated(serviceType, tenantId)) { tpi.tenantId(tenantId); + myPartitionsSearchKey = new ServiceKey(serviceType, tenantId); + } else { + myPartitionsSearchKey = new ServiceKey(serviceType, new TenantId(TenantId.NULL_UUID)); + } + List partitions = myPartitions.get(myPartitionsSearchKey); + if (partitions != null) { + tpi.myPartition(partitions.contains(partition)); + } else { + tpi.myPartition(false); } return tpi.build(); } + private boolean isIsolated(ServiceType serviceType, TenantId tenantId) { + return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + } + private void logServiceInfo(TransportProtos.ServiceInfo server) { TenantId tenantId = getSystemOrIsolatedTenantId(server); if (tenantId.isNullUid()) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 46864af79f..4eb4ca9a69 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -20,13 +20,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; +import java.util.Set; /** * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} */ public interface PartitionService { - List getCurrentPartitions(ServiceType serviceType); + Set getCurrentPartitions(ServiceType serviceType); TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java index 05c5c1586d..1164465ed7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Builder; +import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; @@ -26,13 +27,17 @@ public class TopicPartitionInfo { private final String topic; private final TenantId tenantId; private final Integer partition; + @Getter private final String fullTopicName; + @Getter + private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.myPartition = myPartition; String tmp = topic; if (tenantId != null) { tmp += "." + tenantId.getId().toString(); @@ -40,7 +45,6 @@ public class TopicPartitionInfo { if (partition != null) { tmp += "." + partition; } - this.fullTopicName = tmp; } @@ -56,10 +60,6 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } - public String getFullTopicName() { - return fullTopicName; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java new file mode 100644 index 0000000000..4d02647fd0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.AllArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@AllArgsConstructor +public class TopicPartitionInfoKey { + private ServiceType serviceType; + private TenantId isolatedTenantId; + private int partition; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; + return partition == that.partition && + serviceType == that.serviceType && + Objects.equals(isolatedTenantId, that.isolatedTenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, isolatedTenantId, partition); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 52086b3058..ec08d69be7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -75,7 +75,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null)); + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); subscribed = false; } From 07bdcac0fe2b8b5165cd7823fa53deaba26ea45a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 19:18:27 +0200 Subject: [PATCH 022/102] Refactoring of Rule Engine API --- .../server/actors/ActorSystemContext.java | 19 ++- .../actors/ruleChain/DefaultTbContext.java | 16 ++- .../RuleChainActorMessageProcessor.java | 10 +- .../server/controller/BaseController.java | 19 ++- .../queue/DefaultTbCoreConsumerService.java | 16 ++- .../DefaultTbRuleEngineConsumerService.java | 65 +++++++--- .../server/service/queue/MsgPackCallback.java | 28 +++-- .../TbRuleEngineProcessingDecision.java | 16 +++ .../TbRuleEngineProcessingResult.java | 33 +++++ .../TbRuleEngineProcessingStrategy.java | 7 ++ ...TbRuleEngineProcessingStrategyFactory.java | 118 ++++++++++++++++++ .../state/DefaultDeviceStateService.java | 10 +- .../DefaultSubscriptionManagerService.java | 2 +- .../SubscriptionManagerService.java | 3 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../src/main/resources/thingsboard.yml | 74 ++++++----- .../thingsboard/server/common/msg/TbMsg.java | 6 +- .../service/DefaultTransportService.java | 8 +- 18 files changed, 362 insertions(+), 90 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e5158377cd..64a713b336 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -63,8 +63,13 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; @@ -154,7 +159,6 @@ public class ActorSystemContext { private RuleChainService ruleChainService; @Autowired - @Getter private PartitionService partitionService; @Autowired @@ -402,6 +406,17 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } + public TbQueueProducer> getTbCoreMsgProducer() { + return ruleEngineQueueProvider.getTbCoreMsgProducer(); + } + + public TbQueueProducer> getRuleEngineMsgProducer() { + return ruleEngineQueueProvider.getRuleEngineMsgProducer(); + } + + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, tenantId, entityId); + } public String getServerAddress() { return serviceInfoProvider.getServiceId(); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index a5b5254174..c9273e0a11 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -65,7 +65,10 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -119,7 +122,7 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - return mainCtx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); + return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -151,8 +154,13 @@ class DefaultTbContext implements TbContext { } @Override - public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new QueueToRuleEngineMsg(getTenantId(), msg))); + public void sendTbMsgToRuleEngine(TbMsg tbMsg) { + TenantId tenantId = getTenantId(); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + mainCtx.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index b3c71b3b62..13d370e037 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -68,7 +68,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final PartitionService partitionService; private final TbQueueProducer> producer; private RuleNodeId firstId; @@ -83,7 +82,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.partitionService = systemContext.getPartitionService(); this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); } @@ -234,7 +232,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relations = nodeRoutes.get(originatorNodeId).stream() .filter(r -> contains(envelope.getRelationTypes(), r.getType())) @@ -242,9 +240,9 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(newMsg.getId(), toQueueMsg), callbackWrapper); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index e09b8ca673..bc88544243 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -93,6 +93,12 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -185,6 +191,12 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; + @Autowired + protected PartitionService partitionService; + + @Autowired + protected TbCoreQueueProvider coreQueueProvider; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -662,7 +674,12 @@ public abstract class BaseController { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) , null, null, null); - actorService.onMsg(new SendToClusterMsg(entityId, new QueueToRuleEngineMsg(user.getTenantId(), tbMsg))); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, user.getTenantId(), entityId); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(user.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(user.getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + coreQueueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c081fc526a..30148dfe08 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,7 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -104,11 +105,13 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { if (msgs.isEmpty()) { continue; } - ConcurrentMap> ackMap = msgs.stream().collect( + ConcurrentMap> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - ackMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, successMap, failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToDeviceActorMsg()) { @@ -130,7 +133,8 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } }); if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } catch (Exception e) { @@ -182,7 +186,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); } else if (msg.hasTsUpdate()) { TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); - subscriptionManagerService.onTimeseriesDataUpdate( + subscriptionManagerService.onTimeSeriesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 1f84c7cc98..986d858043 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -36,11 +36,16 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -64,10 +69,12 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS private final ActorSystemContext actorContext; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + private final TbRuleEngineProcessingStrategyFactory factory; private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + this.factory = factory; this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); this.actorContext = actorContext; } @@ -75,6 +82,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PostConstruct public void init() { this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.factory.newInstance(); } @Override @@ -94,29 +102,46 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS if (msgs.isEmpty()) { continue; } - ConcurrentMap> ackMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - ackMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); - try { - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); - } else { - callback.onSuccess(); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(); + TbRuleEngineProcessingDecision decision = null; + boolean firstAttempt = true; + while (!stopped && (firstAttempt || !decision.isCommit())) { + ConcurrentMap> allMap; + if (firstAttempt) { + allMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + firstAttempt = false; + } else { + allMap = decision.getReprocessMap(); + } + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + allMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); + try { + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); } - } catch (Throwable e) { - callback.onFailure(e); + }); + + boolean timeout = false; + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + timeout = true; } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } consumer.commit(); } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); + log.warn("Failed to process messages from queue.", e); try { Thread.sleep(pollDuration); } catch (InterruptedException e2) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d616bbef16..420c61d5d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,25 +20,37 @@ import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class MsgPackCallback implements TbMsgCallback { +public class MsgPackCallback implements TbMsgCallback { private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap> ackMap; + private final ConcurrentMap ackMap; + private final ConcurrentMap successMap; + private final ConcurrentMap failedMap; private final UUID id; - public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, ConcurrentMap> ackMap) { + public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap successMap, + ConcurrentMap failedMap) { this.id = id; this.processingTimeoutLatch = processingTimeoutLatch; this.ackMap = ackMap; + this.successMap = successMap; + this.failedMap = failedMap; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - if (ackMap.remove(id) != null && ackMap.isEmpty()) { + T msg = ackMap.remove(id); + if (msg != null) { + successMap.put(id, msg); + } + if (msg != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } } @@ -46,8 +58,10 @@ public class MsgPackCallback i @Override public void onFailure(Throwable t) { log.trace("[{}] ON FAILURE", id); - TbProtoQueueMsg message = ackMap.remove(id); - log.warn("Failed to process message: {}", message.getValue(), t); + T msg = ackMap.remove(id); + if (msg != null) { + failedMap.put(id, msg); + } if (ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java new file mode 100644 index 0000000000..b116fd7443 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -0,0 +1,16 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.Data; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +@Data +public class TbRuleEngineProcessingDecision { + + private final boolean commit; + private final ConcurrentMap> reprocessMap; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java new file mode 100644 index 0000000000..601a3e9a9b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -0,0 +1,33 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.Getter; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +public class TbRuleEngineProcessingResult { + + @Getter + private boolean success; + @Getter + private boolean timeout; + @Getter + private ConcurrentMap> pendingMap; + @Getter + private ConcurrentMap> successMap; + @Getter + private ConcurrentMap> failureMap; + + public TbRuleEngineProcessingResult(boolean timeout, + ConcurrentMap> pendingMap, + ConcurrentMap> successMap, + ConcurrentMap> failureMap) { + this.timeout = timeout; + this.pendingMap = pendingMap; + this.successMap = successMap; + this.failureMap = failureMap; + this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java new file mode 100644 index 0000000000..c644766892 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -0,0 +1,7 @@ +package org.thingsboard.server.service.queue.processing; + +public interface TbRuleEngineProcessingStrategy { + + TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java new file mode 100644 index 0000000000..336baf5360 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -0,0 +1,118 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class TbRuleEngineProcessingStrategyFactory { + + @Value("${queue.rule_engine.strategy.type}") + private String strategyType; + @Value("${queue.rule_engine.strategy.retries:3}") + private int maxRetries; + @Value("${queue.rule_engine.strategy.failure_percentage:0}") + private double maxAllowedFailurePercentage; + @Value("${queue.rule_engine.strategy.pause_between_retries:3}") + private long pauseBetweenRetries; + + + public TbRuleEngineProcessingStrategy newInstance() { + switch (strategyType) { + case "SKIP_ALL": + return new SkipStrategy(); + case "RETRY_ALL": + return new RetryStrategy(true, true, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_FAILED": + return new RetryStrategy(false, true, false, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_TIMED_OUT": + return new RetryStrategy(false, false, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_FAILED_AND_TIMED_OUT": + return new RetryStrategy(false, true, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + strategyType + " is not supported!"); + } + } + + private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final boolean retrySuccessful; + private final boolean retryFailed; + private final boolean retryTimeout; + private final int maxRetries; + private final double maxAllowedFailurePercentage; + private final long pauseBetweenRetries; + + private int initialTotalCount; + private int retryCount; + + public RetryStrategy(boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, int maxRetries, double maxAllowedFailurePercentage, long pauseBetweenRetries) { + this.retrySuccessful = retrySuccessful; + this.retryFailed = retryFailed; + this.retryTimeout = retryTimeout; + this.maxRetries = maxRetries; + this.maxAllowedFailurePercentage = maxAllowedFailurePercentage; + this.pauseBetweenRetries = pauseBetweenRetries; + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (result.isSuccess()) { + return new TbRuleEngineProcessingDecision(true, null); + } else { + if (retryCount == 0) { + initialTotalCount = result.getPendingMap().size() + result.getFailureMap().size() + result.getSuccessMap().size(); + } + retryCount++; + double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); + if (maxRetries > 0 && retryCount > maxRetries) { + log.info("Skip reprocess of the rule engine pack due to max retries"); + return new TbRuleEngineProcessingDecision(true, null); + } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { + log.info("Skip reprocess of the rule engine pack due to max allowed failure percentage"); + return new TbRuleEngineProcessingDecision(true, null); + } else { + ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); + if (retryFailed) { + result.getFailureMap().forEach(toReprocess::put); + } + if (retryTimeout) { + result.getPendingMap().forEach(toReprocess::put); + } + if (retrySuccessful) { + result.getSuccessMap().forEach(toReprocess::put); + } + log.info("Going to reprocess {} messages", toReprocess.size()); + //TODO: 2.5 Log most popular rule nodes by error count; + if (log.isTraceEnabled()) { + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); + } + if (pauseBetweenRetries > 0) { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return new TbRuleEngineProcessingDecision(false, toReprocess); + } + } + } + } + + private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + log.info("Skip reprocess of the rule engine pack"); + return new TbRuleEngineProcessingDecision(true, null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 816479b1e2..84fa1a7ac2 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -502,7 +503,12 @@ public class DefaultDeviceStateService implements DeviceStateService { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) , null, null, null); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new QueueToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + queueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index be186e0ddd..adff35fa51 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -187,7 +187,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index d4d9fa5841..aa1824c855 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -31,7 +31,8 @@ public interface SubscriptionManagerService extends ApplicationListener ts, TbMsgCallback callback); + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 58ed13c1e6..ab2803345a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onTimeseriesDataUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8a4d735d26..c73e623d56 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -181,27 +181,27 @@ cassandra: # SQL configuration parameters sql: - # Specify batch size for persisting attribute updates - attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" - ts: - batch_size: "${SQL_TS_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" - ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" - # Specify whether to remove null characters from strValue of attributes and timeseries before insert - remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" - postgres: - # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. - ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" - timescale: - # Specify Interval size for new data chunks storage. - chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" + # Specify batch size for persisting attribute updates + attributes: + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" + ts: + batch_size: "${SQL_TS_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" + ts_latest: + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" + # Specify whether to remove null characters from strValue of attributes and timeseries before insert + remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" + postgres: + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. + ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" + timescale: + # Specify Interval size for new data chunks storage. + chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" # Actor system parameters actors: @@ -330,19 +330,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -551,6 +551,12 @@ queue: poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure_percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause_between_retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 9b43169737..062eb02749 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; import lombok.Builder; @@ -73,7 +74,10 @@ public final class TbMsg implements Serializable { log.warn("[{}] Created message with empty callback: {}", originator, type); this.callback = TbMsgCallback.EMPTY; } + } + public static ByteString toByteString(TbMsg msg) { + return ByteString.copyFrom(toByteArray(msg)); } public static byte[] toByteArray(TbMsg msg) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 1a80aa4a6e..42d3a534a9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -244,7 +244,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("ts", tsKv.getTs() + ""); JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), - deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); + deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); sendToRuleEngine(tenantId, tbMsg, packCallback); } } @@ -261,7 +261,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("deviceName", sessionInfo.getDeviceName()); metaData.putValue("deviceType", sessionInfo.getDeviceType()); TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, - TbMsgDataType.JSON, gson.toJson(json), null, null, null); + TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } @@ -488,7 +488,7 @@ public class DefaultTransportService implements TransportService { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(tbMsg))) + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); From 7531a26e617092c437b9df6c8d13b22511eac1ab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 Mar 2020 22:11:47 +0200 Subject: [PATCH 023/102] Implementation of RPC Call support --- .../server/actors/ActorSystemContext.java | 49 ++-- .../server/actors/app/AppActor.java | 6 +- .../server/actors/device/DeviceActor.java | 6 + .../device/DeviceActorMessageProcessor.java | 21 +- .../actors/ruleChain/DefaultTbContext.java | 46 +--- .../actors/ruleChain/RuleChainActor.java | 4 +- .../RuleChainActorMessageProcessor.java | 16 +- .../RuleNodeActorMessageProcessor.java | 6 +- .../server/actors/service/ActorService.java | 17 +- .../server/actors/service/ComponentActor.java | 6 +- .../actors/service/DefaultActorService.java | 81 +------ .../actors/shared/ComponentMsgProcessor.java | 11 +- .../server/actors/tenant/TenantActor.java | 28 ++- .../server/controller/BaseController.java | 21 +- .../server/controller/DeviceController.java | 15 +- .../server/controller/RpcController.java | 12 +- .../controller/RuleChainController.java | 12 +- .../controller/TelemetryController.java | 11 +- .../server/controller/TenantController.java | 2 +- .../encoding/DataDecodingEncodingService.java | 5 - .../service/encoding/ProtoWithFSTService.java | 24 +- .../queue/DefaultTbClusterService.java | 125 ++++++++++ .../queue/DefaultTbCoreConsumerService.java | 156 ++++++++++--- .../DefaultTbRuleEngineConsumerService.java | 103 ++++++-- .../server/service/queue/MsgPackCallback.java | 2 +- .../service/queue/TbClusterService.java | 37 +++ .../TbRuleEngineProcessingDecision.java | 15 ++ .../TbRuleEngineProcessingResult.java | 15 ++ .../TbRuleEngineProcessingStrategy.java | 15 ++ ...TbRuleEngineProcessingStrategyFactory.java | 15 ++ .../service/rpc/DefaultDeviceRpcService.java | 219 ------------------ .../rpc/DefaultTbCoreDeviceRpcService.java | 203 ++++++++++++++++ .../rpc/DefaultTbRuleEngineRpcService.java | 169 ++++++++++++++ .../server/service/rpc/DeviceRpcService.java | 41 ---- .../service/rpc/TbCoreDeviceRpcService.java | 57 +++++ .../rpc/TbRuleEngineDeviceRpcService.java | 16 +- .../rpc/ToDeviceRpcRequestActorMsg.java | 2 +- .../state/DefaultDeviceStateService.java | 58 ++--- .../DefaultSubscriptionManagerService.java | 37 ++- ...=> DefaultTbLocalSubscriptionService.java} | 12 +- ...e.java => TbLocalSubscriptionService.java} | 2 +- .../DefaultTelemetrySubscriptionService.java | 10 +- .../DefaultTelemetryWebSocketService.java | 4 +- .../DefaultTbCoreToTransportService.java | 8 +- application/src/main/proto/cluster.proto | 74 ------ .../src/main/resources/thingsboard.yml | 5 + ...AbstractRuleEngineFlowIntegrationTest.java | 6 +- ...actRuleEngineLifecycleIntegrationTest.java | 3 +- .../ConsistentHashParitionServiceTest.java | 4 +- .../server/common/msg/MsgType.java | 5 +- .../thingsboard/server/common/msg/TbMsg.java | 2 +- .../msg/plugin/ComponentLifecycleMsg.java | 2 +- .../PartitionChangeMsg.java} | 15 +- .../server/common/msg/queue}/ServiceKey.java | 2 +- .../server/common/msg/queue}/ServiceType.java | 2 +- .../common/msg/queue}/TopicPartitionInfo.java | 2 +- .../server/queue/TbQueueConsumer.java | 2 +- .../server/queue/TbQueueProducer.java | 2 +- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../DefaultTbQueueResponseTemplate.java | 2 +- .../discovery/ClusterTopologyChangeEvent.java | 1 + .../ConsistentHashPartitionService.java | 48 ++-- .../DefaultTbServiceInfoProvider.java | 1 + .../queue/discovery/PartitionChangeEvent.java | 2 + .../queue/discovery/PartitionService.java | 13 +- .../discovery/TbServiceInfoProvider.java | 1 + .../discovery/TopicPartitionInfoKey.java | 1 + .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- .../queue/kafka/TBKafkaProducerTemplate.java | 2 +- .../queue/memory/InMemoryTbQueueConsumer.java | 2 +- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../InMemoryMonolithQueueProvider.java | 32 ++- ... => InMemoryTbTransportQueueProvider.java} | 10 +- .../provider/KafkaMonolithQueueProvider.java | 68 ++++-- .../provider/KafkaTbCoreQueueProvider.java | 77 ++++-- .../KafkaTbRuleEngineQueueProvider.java | 60 ++++- ...ava => KafkaTbTransportQueueProvider.java} | 14 +- .../provider/TbCoreQueueProducerProvider.java | 74 ++++++ .../queue/provider/TbCoreQueueProvider.java | 22 ++ .../provider/TbQueueProducerProvider.java | 66 ++++++ .../TbRuleEngineProducerProvider.java | 75 ++++++ .../provider/TbRuleEngineQueueProvider.java | 23 ++ .../TbTransportQueueProducerProvider.java | 70 ++++++ ...der.java => TbTransportQueueProvider.java} | 2 +- common/queue/src/main/proto/queue.proto | 20 +- .../transport/coap/CoapTransportService.java | 6 - .../service/DefaultTransportService.java | 16 +- .../api/RuleEngineDeviceRpcRequest.java | 5 +- .../rule/engine/api/RuleEngineRpcService.java | 4 +- .../thingsboard/rule/engine/api/TbNode.java | 4 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 4 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 2 +- .../rule/engine/rpc/TbSendRPCRequestNode.java | 15 +- 93 files changed, 1720 insertions(+), 874 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java rename common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java => application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java (59%) rename application/src/main/java/org/thingsboard/server/service/subscription/{DefaultLocalSubscriptionService.java => DefaultTbLocalSubscriptionService.java} (96%) rename application/src/main/java/org/thingsboard/server/service/subscription/{LocalSubscriptionService.java => TbLocalSubscriptionService.java} (96%) delete mode 100644 application/src/main/proto/cluster.proto rename common/message/src/main/java/org/thingsboard/server/common/msg/{cluster/ClusterEventMsg.java => queue/PartitionChangeMsg.java} (72%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/ServiceKey.java (96%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/ServiceType.java (94%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/TopicPartitionInfo.java (97%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryTransportQueueProvider.java => InMemoryTbTransportQueueProvider.java} (90%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTransportQueueProvider.java => KafkaTbTransportQueueProvider.java} (91%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TransportQueueProvider.java => TbTransportQueueProvider.java} (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 64a713b336..03b65e7ccc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; @@ -63,14 +65,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -78,7 +75,9 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; @@ -163,7 +162,11 @@ public class ActorSystemContext { @Autowired @Getter - private TbRuleEngineQueueProvider ruleEngineQueueProvider; + private TbClusterService clusterService; + + @Autowired + @Getter + private TbQueueProducerProvider producerProvider; @Autowired @Getter @@ -197,10 +200,6 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; - @Autowired - @Getter - private DeviceRpcService deviceRpcService; - @Autowired @Getter private JsInvokeService jsSandbox; @@ -246,6 +245,22 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + /** + * The following Service will be null if we operate in tb-core mode + */ + @Lazy + @Getter + @Autowired(required = false) + private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; + + /** + * The following Service will be null if we operate in tb-rule-engine mode + */ + @Lazy + @Getter + @Autowired(required = false) + private TbCoreDeviceRpcService tbCoreDeviceRpcService; + @Lazy @Autowired @Getter @@ -406,14 +421,6 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } - public TbQueueProducer> getTbCoreMsgProducer() { - return ruleEngineQueueProvider.getTbCoreMsgProducer(); - } - - public TbQueueProducer> getRuleEngineMsgProducer() { - return ruleEngineQueueProvider.getRuleEngineMsgProducer(); - } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return partitionService.resolve(serviceType, tenantId, entityId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 9d80d63b2a..c096238a8b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -39,7 +39,9 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; @@ -81,7 +83,7 @@ public class AppActor extends RuleChainManagerActor { case SEND_TO_CLUSTER_MSG: onPossibleClusterMsg((SendToClusterMsg) msg); break; - case CLUSTER_EVENT_MSG: + case PARTITION_CHANGE_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: @@ -131,7 +133,7 @@ public class AppActor extends RuleChainManagerActor { // systemContext.getRpcService().tell( // systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); // } else { - self().tell(msg.getMsg(), ActorRef.noSender()); + self().tell(msg.getMsg(), ActorRef.noSender()); // } } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 547e484652..3a15240f33 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; @@ -48,6 +49,11 @@ public class DeviceActor extends ContextAwareActor { } } + @Override + public void postStop() { + + } + @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 4713662067..aff6200199 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,13 +16,10 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; -import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -38,11 +35,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.gen.transport.TransportProtos; @@ -52,8 +46,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestM import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; @@ -63,14 +55,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.common.transport.util.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -103,8 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map toDeviceRpcPendingMap; private final Map toServerRpcPendingMap; - private final Gson gson = new Gson(); - private int rpcSeq = 0; private String deviceName; private String deviceType; @@ -162,7 +150,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -183,7 +171,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } @@ -215,7 +203,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -417,7 +405,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -689,4 +677,5 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index c9273e0a11..7c831929ca 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -48,9 +48,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -67,8 +65,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -155,12 +153,7 @@ class DefaultTbContext implements TbContext { @Override public void sendTbMsgToRuleEngine(TbMsg tbMsg) { - TenantId tenantId = getTenantId(); - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); - mainCtx.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + mainCtx.getClusterService().onToRuleEngineMsg(getTenantId(), tbMsg.getOriginator(), tbMsg); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { @@ -351,36 +344,7 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return new RuleEngineRpcService() { - @Override - public void sendRpcReply(DeviceId deviceId, int requestId, String body) { - mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); - } - - @Override - public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - //TODO 2.5 -// ServerAddress requestOriginAddress; -// if (!StringUtils.isEmpty(src.getOriginHost())) { -// requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); -// } else { -// requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); -// } -// mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - }; + return mainCtx.getTbRuleEngineDeviceRpcService(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 83cde28a45..1b9923139a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; @@ -51,7 +52,8 @@ public class RuleChainActor extends ComponentActor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); + this.producer = systemContext.getProducerProvider().getRuleEngineMsgProducer(); } @Override @@ -153,8 +151,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index b4bda98f7a..2dd6764407 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.dao.rule.RuleChainService; /** @@ -84,9 +84,9 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor status = system.terminate(); @@ -108,68 +107,10 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onMsg(TbActorMsg msg) { - appActor.tell(msg, ActorRef.noSender()); - } - - //TODO 2.5 -// @Override -// public void onServerAdded(ServerInstance server) { -// log.trace("Processing onServerAdded msg: {}", server); -// broadcast(new ClusterEventMsg(server.getServerAddress(), true)); -// } -// -// @Override -// public void onServerUpdated(ServerInstance server) { -// //Do nothing -// } -// -// @Override -// public void onServerRemoved(ServerInstance server) { -// log.trace("Processing onServerRemoved msg: {}", server); -// broadcast(new ClusterEventMsg(server.getServerAddress(), false)); -// } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - @Override - public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { - DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - @Override - public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { - log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); - DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - public void broadcast(ToAllNodesMsg msg) { - actorContext.getEncodingService().encode(msg); - //TODO 2.5 -// rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage -// .newBuilder() -// .setPayload(ByteString -// .copyFrom(actorContext.getEncodingService().encode(msg))) -// .setMessageType(CLUSTER_ACTOR_MESSAGE) -// .build())); - appActor.tell(msg, ActorRef.noSender()); - } - - private void broadcast(ClusterEventMsg msg) { - this.appActor.tell(msg, ActorRef.noSender()); - this.rpcManagerActor.tell(msg, ActorRef.noSender()); - } - @Value("${cluster.stats.enabled:false}") private boolean statsEnabled; + //TODO 2.5 private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); @@ -258,8 +199,4 @@ public class DefaultActorService implements ActorService { // rpcManagerActor.tell(msg, ActorRef.noSender()); // } - @Override - public void onDeviceAdded(Device device) { - deviceStateService.onDeviceAdded(device); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 2fdc918f8e..1443c61e16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,20 +16,13 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; -import akka.event.LoggingAdapter; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; - -import javax.annotation.Nullable; -import java.util.function.Consumer; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -50,7 +43,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; + public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index d47591f252..80629e5927 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -41,9 +41,14 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; +import java.util.List; +import java.util.stream.Collectors; + public class TenantActor extends RuleChainManagerActor { private final TenantId tenantId; @@ -79,8 +84,21 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { - case CLUSTER_EVENT_MSG: - broadcast(msg); + case PARTITION_CHANGE_MSG: + PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; + ServiceType serviceType = partitionChangeMsg.getServiceKey().getServiceType(); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + //To Rule Chain Actors + broadcast(msg); + } else if (ServiceType.TB_CORE.equals(serviceType)) { + //To Device Actors + List repartitionedDevices = + deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); + for (DeviceId deviceId : repartitionedDevices) { + ActorRef deviceActor = deviceActors.remove(deviceId); + context().stop(deviceActor); + } + } break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); @@ -106,6 +124,10 @@ public class TenantActor extends RuleChainManagerActor { return true; } + private boolean isMyPartition(DeviceId deviceId) { + return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); + } + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { @@ -169,7 +191,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.warn("Removed actor was not found in the device map!"); + log.debug("Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index bc88544243..9c8ba94ccf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -27,7 +27,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; @@ -71,8 +70,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -93,13 +90,10 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -168,7 +162,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected ActorService actorService; + protected TbClusterService tbClusterService; @Autowired protected RelationService relationService; @@ -195,7 +189,7 @@ public abstract class BaseController { protected PartitionService partitionService; @Autowired - protected TbCoreQueueProvider coreQueueProvider; + protected TbQueueProducerProvider producerProvider; @Value("${server.log_controller_error_stack_trace}") @Getter @@ -674,12 +668,7 @@ public abstract class BaseController { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) , null, null, null); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, user.getTenantId(), entityId); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(user.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(user.getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - coreQueueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + tbClusterService.onToRuleEngineMsg(user.getTenantId(), entityId, tbMsg); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index d147d27b40..562d46d9f0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -30,6 +30,8 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -94,12 +96,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - actorService - .onDeviceNameOrTypeUpdate( - savedDevice.getTenantId(), - savedDevice.getId(), - savedDevice.getName(), - savedDevice.getType()); + tbClusterService.onToCoreMsg(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType())); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -252,7 +250,9 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); + + tbClusterService.onToCoreMsg(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId())); + logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); @@ -425,6 +425,7 @@ public class DeviceController extends BaseController { deferredResult.setResult(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } } + @Override public void onFailure(Throwable t) { deferredResult.setErrorResult(t); diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index a8298ce82b..2d747f32e6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.FutureCallback; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -31,7 +32,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -43,22 +43,18 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. @@ -72,7 +68,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private DeviceRpcService deviceRpcService; + private TbCoreDeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -116,7 +112,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 3115136713..9dc1974e2a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -131,7 +131,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -162,7 +162,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -170,7 +170,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -204,7 +204,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -255,9 +255,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index e534ebf7c4..b2b3809dd2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.service.security.AccessValidator; @@ -362,9 +361,8 @@ public class TelemetryController extends BaseController { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify)); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -398,9 +396,8 @@ public class TelemetryController extends BaseController { logAttributesUpdated(user, entityId, scope, attributes, null); if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes)); } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 8d95e78cfa..a89658ba0c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -95,7 +95,7 @@ public class TenantController extends BaseController { checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index a5d3ab465e..4a781b8673 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -27,8 +25,5 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); - ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 45bb9f78fd..8d89059488 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,25 +15,19 @@ */ package org.thingsboard.server.service.encoding; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - - @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + @Override public Optional decode(byte[] byteArray) { try { @@ -42,7 +36,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -51,18 +45,4 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } - @Override - public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg) { - return ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .setPayload(ByteString.copyFrom(encode(msg))).build(); - - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java new file mode 100644 index 0000000000..5851fd919c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.HashSet; +import java.util.Set; + +@Service +@Slf4j +public class DefaultTbClusterService implements TbClusterService { + + protected TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final DataDecodingEncodingService encodingService; + + public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.encodingService = encodingService; + } + + @Override + public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + } + + @Override + public void onToCoreMsg(ToDeviceActorNotificationMsg msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + byte[] msgBytes = encodingService.encode(msg); + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), null); + } + + @Override + public void onToCoreMsg(String serviceId, FromDeviceRpcResponse response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + } + + @Override + public void onToRuleEngineMsg(String serviceId, FromDeviceRpcResponse response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + private void broadcast(ComponentLifecycleMsg msg) { + byte[] msgBytes = encodingService.encode(msg); + TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + TbQueueProducer> toCoreProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); + for (String serviceId : tbCoreServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + } + // No need to push notifications twice + tbRuleEngineServices.removeAll(tbCoreServices); + } + for (String serviceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 30148dfe08..4e2387082b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,26 +24,32 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.subscription.LocalSubscriptionService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,64 +74,85 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final DeviceStateService stateService; - private final LocalSubscriptionService localSubscriptionService; + private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; - private final TbQueueConsumer> consumer; + private final DataDecodingEncodingService encodingService; + private final TbQueueConsumer> mainConsumer; + private final TbQueueConsumer> nfConsumer; + private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, LocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService) { - this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, + DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService) { + this.mainConsumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.nfConsumer = tbCoreQueueProvider.getToCoreNotificationsMsgConsumer(); this.actorContext = actorContext; this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; + this.encodingService = encodingService; + this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; } @PostConstruct public void init() { this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-notifications-consumer")); } @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_CORE) { log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.consumer.subscribe(partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); } } @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", mainConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + private void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = mainConsumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } ConcurrentMap> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> successMap = new ConcurrentHashMap<>(); ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, successMap, failedMap); + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToDeviceActorMsg()) { + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); } else if (toCoreMsg.hasDeviceStateServiceMsg()) { log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); - } else if (toCoreMsg.hasToSubscriptionMgrMsg()) { - log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); - forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); - } else if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -136,7 +163,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } - consumer.commit(); + mainConsumer.commit(); } catch (Exception e) { log.warn("Failed to obtain messages from queue.", e); try { @@ -150,6 +177,67 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } + private void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + ToCoreNotificationMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); + } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + log.info("Tb Core Notifications Consumer stopped."); + }); + } + + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbMsgCallback callback) { + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); + callback.onSuccess(); + } + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { @@ -160,15 +248,21 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { @PreDestroy public void destroy() { stopped = true; - if (consumer != null) { - consumer.unsubscribe(); + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } } - private void forwardToLocalSubMgrService(TransportProtos.LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { + private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { if (msg.hasSubUpdate()) { localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); } else { @@ -176,22 +270,22 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - private void forwardToSubMgrService(TransportProtos.SubscriptionMgrMsgProto msg, TbMsgCallback callback) { + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbMsgCallback callback) { if (msg.hasAttributeSub()) { subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); } else if (msg.hasTelemetrySub()) { subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); } else if (msg.hasSubClose()) { - TransportProtos.TbSubscriptionCloseProto closeProto = msg.getSubClose(); + TbSubscriptionCloseProto closeProto = msg.getSubClose(); subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); } else if (msg.hasTsUpdate()) { - TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); subscriptionManagerService.onTimeSeriesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); } else if (msg.hasAttrUpdate()) { - TransportProtos.TbAttributeUpdateProto proto = msg.getAttrUpdate(); + TbAttributeUpdateProto proto = msg.getAttrUpdate(); subscriptionManagerService.onAttributesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), @@ -201,7 +295,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { if (statsEnabled) { stats.log(deviceStateServiceMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 986d858043..b9d2b98d04 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,16 +26,19 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; @@ -44,6 +47,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -67,21 +71,28 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS private boolean statsEnabled; private final ActorSystemContext actorContext; - private final TbQueueConsumer> consumer; + private final TbQueueConsumer> mainConsumer; + private final TbQueueConsumer> nfConsumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; + private final DataDecodingEncodingService encodingService; private volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext, + DataDecodingEncodingService encodingService) { this.factory = factory; - this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); + this.mainConsumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); + this.nfConsumer = tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer(); this.actorContext = actorContext; + this.encodingService = encodingService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-notifications-consumer")); this.factory.newInstance(); } @@ -89,16 +100,70 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.consumer.subscribe(partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); } } @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + private void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + ToRuleEngineNotificationMsg toRuleEngineMsg = msg.getValue(); + if (toRuleEngineMsg.getComponentLifecycleMsg() != null && !toRuleEngineMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toRuleEngineMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = mainConsumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } @@ -106,7 +171,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { - ConcurrentMap> allMap; + ConcurrentMap> allMap; if (firstAttempt) { allMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); @@ -114,14 +179,14 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } else { allMap = decision.getReprocessMap(); } - ConcurrentMap> successMap = new ConcurrentHashMap<>(); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); try { - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); @@ -139,7 +204,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } - consumer.commit(); + mainConsumer.commit(); } catch (Exception e) { log.warn("Failed to process messages from queue.", e); try { @@ -171,11 +236,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PreDestroy public void destroy() { stopped = true; - if (consumer != null) { - consumer.unsubscribe(); + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index 420c61d5d7..72c5c10458 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java new file mode 100644 index 0000000000..952b88e4d3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +public interface TbClusterService { + + void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg msg); + + void onToCoreMsg(ToDeviceActorNotificationMsg msg); + + void onToCoreMsg(String targetServiceId, FromDeviceRpcResponse response); + + void onToRuleEngineMsg(String targetServiceId, FromDeviceRpcResponse response); + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java index b116fd7443..4a4829f0c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index 601a3e9a9b..c17cf54bca 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java index c644766892..cfba85f5dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; public interface TbRuleEngineProcessingStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 336baf5360..8d45959df3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java deleted file mode 100644 index d8f12b8f0a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -public class DefaultDeviceRpcService implements DeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - @Autowired - private DeviceService deviceService; - - @Autowired - @Lazy - private ActorService actorService; - - private ScheduledExecutorService rpcCallBackExecutor; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - @PostConstruct - public void initExecutor() { - rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (rpcCallBackExecutor != null) { - rpcCallBackExecutor.shutdownNow(); - } - } - - @Override - public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - //TODO 2.5 - if (true) {//routingService.getCurrentServer().equals(requestOriginAddress) - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } else { - ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); - builder.setRequestIdMSB(response.getId().getMostSignificantBits()); - builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); - response.getResponse().ifPresent(builder::setResponse); - if (response.getError().isPresent()) { - builder.setError(response.getError().get().ordinal()); - } else { - builder.setError(-1); - } - //TODO 2.5 -// rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); - } - } - - @Override - public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId, localToDeviceRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = localToDeviceRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.FromDeviceRPCResponseProto proto; - try { - proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - //TODO 2.5 -// processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); - } - - @Override - public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { - ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); - forward(deviceId, rpcMsg); - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - //TODO 2.5 -// metaData.putValue("originHost", routingService.getCurrentServer().getHost()); -// metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, null); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new QueueToRuleEngineMsg(msg.getTenantId(), tbMsg))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - //TODO 2.5 -// ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); -// log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); -// forward(msg.getDeviceId(), rpcMsg); - } - - private void forward(DeviceId deviceId, T msg) { - actorService.onMsg(new SendToClusterMsg(deviceId, msg)); - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - rpcCallBackExecutor.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = requestsMap.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java new file mode 100644 index 0000000000..ed55ee11df --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -0,0 +1,203 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import akka.actor.ActorRef; +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + private final DeviceService deviceService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + private final ActorSystemContext actorContext; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbRuleEngineRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, + ActorSystemContext actorContext) { + this.deviceService = deviceService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + this.actorContext = actorContext; + } + + @Autowired + public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { + this.tbRuleEngineRpcService = tbRuleEngineRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleToRuleEngineTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { + ToDeviceRpcRequest request = rpcMsg.getMsg(); + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, rpcMsg); + actorContext.getAppActor().tell(rpcMsg, ActorRef.noSender()); + scheduleToDeviceTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); + if (request != null) { + sendRpcResponseToTbRuleEngine(request.getServiceId(), response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbRuleEngineRpcService.isPresent()) { + tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.onToRuleEngineMsg(originServiceId, response); + } + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originServiceId", serviceId); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, null); + clusterService.onToRuleEngineMsg(msg.getTenantId(), msg.getDeviceId(), tbMsg); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); + localToDeviceRpcRequests.remove(requestId); + }, timeout, TimeUnit.MILLISECONDS); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java new file mode 100644 index 0000000000..6481f1c2b6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@Slf4j +public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { + + private final PartitionService partitionService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbCoreRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbRuleEngineRpcService(PartitionService partitionService, + TbClusterService clusterService, + TbServiceInfoProvider serviceInfoProvider) { + this.partitionService = partitionService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Autowired + public void setTbCoreRpcService(Optional tbCoreRpcService) { + this.tbCoreRpcService = tbCoreRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body) { +// TODO 2.5 + } + + @Override + public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + forwardRpcRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + sendRpcResponseToTbCore(src.getOriginServiceId(), response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + + @Override + public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(response)); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + toDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId); + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); + if (tpi.isMyPartition()) { + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); + clusterService.onToCoreMsg(rpcMsg); + } + } + + private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.onToCoreMsg(originServiceId, response); + } + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); + } + }, timeout, TimeUnit.MILLISECONDS); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java deleted file mode 100644 index feb5d58c3d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 16.04.18. - */ -public interface DeviceRpcService { - - void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); - - void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); - - void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); - - void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java new file mode 100644 index 0000000000..896745c48e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Handles REST API calls that contain RPC requests to Device. + */ +public interface TbCoreDeviceRpcService { + + /** + * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. + * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} + * + * @param request the RPC request + * @param responseConsumer the consumer of the RPC response + */ + void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); + + /** + * Handles the RPC response from the Rule Engine. + * + * @param response the RPC response + */ + void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); + + /** + * Forwards the RPC request from Rule Engine to Device Actor + * + * @param request the RPC request message + */ + void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java similarity index 59% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java rename to application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java index ed1bbf18b5..3d4f23a796 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java @@ -13,12 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport; +package org.thingsboard.server.service.rpc; -import org.thingsboard.server.common.data.Device; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; -public interface SessionMsgProcessor { +/** + * Created by ashvayka on 16.04.18. + */ +public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { - void onDeviceAdded(Device device); + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDevice(FromDeviceRpcResponse response); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index ba537aed25..3a1ea599e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -35,7 +35,7 @@ import java.util.Optional; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final ServerAddress serverAddress; + private final String serviceId; @Getter private final ToDeviceRpcRequest msg; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 84fa1a7ac2..91a49aae1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +32,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -50,19 +49,17 @@ import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -106,30 +103,13 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - @Autowired - private TenantService tenantService; - - @Autowired - private DeviceService deviceService; - - @Autowired - private AttributesService attributesService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - private TbCoreQueueProvider queueProvider; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TelemetrySubscriptionService tsSubService; + private final TenantService tenantService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService tsService; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final TelemetrySubscriptionService tsSubService; @Value("${state.defaultInactivityTimeoutInSec}") @Getter @@ -155,6 +135,18 @@ public class DefaultDeviceStateService implements DeviceStateService { private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, + AttributesService attributesService, TimeseriesService tsService, + TbQueueProducerProvider producerProvider, PartitionService partitionService, TelemetrySubscriptionService tsSubService) { + this.tenantService = tenantService; + this.deviceService = deviceService; + this.attributesService = attributesService; + this.tsService = tsService; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.tsSubService = tsSubService; + } + @PostConstruct public void init() { // Should be always single threaded due to absence of locks. @@ -429,7 +421,7 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setUpdated(updated); builder.setDeleted(deleted); TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - queueProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); } @@ -508,7 +500,7 @@ public class DefaultDeviceStateService implements DeviceStateService { .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - queueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index adff35fa51..d6e04d6dd3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -22,7 +22,6 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; @@ -34,21 +33,22 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -86,16 +86,16 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer private TbServiceInfoProvider serviceInfoProvider; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired - private LocalSubscriptionService localSubscriptionService; + private TbLocalSubscriptionService localSubscriptionService; @Autowired private DeviceStateService deviceStateService; @Autowired - private ActorService actorService; + private TbClusterService clusterService; private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); @@ -104,13 +104,13 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer private ExecutorService tsCallBackExecutor; private String serviceId; - private TbQueueProducer> toCoreProducer; + private TbQueueProducer> toCoreNotificationsProducer; @PostConstruct public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); serviceId = serviceInfoProvider.getServiceId(); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); } @PreDestroy @@ -241,9 +241,8 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(notificationMsg); + clusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))); } } callback.onSuccess(); @@ -263,7 +262,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); } else { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); - toCoreProducer.send(tpi, toProto(s, subscriptionUpdate), null); + toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); } } }); @@ -309,7 +308,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer }); if (!missedUpdates.isEmpty()) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); } }, e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); @@ -333,7 +332,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer missedUpdates -> { if (missedUpdates != null && !missedUpdates.isEmpty()) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); } }, e -> log.error("Failed to fetch missed updates.", e), @@ -341,7 +340,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } - private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); builder.setSessionId(subscription.getSessionId()); @@ -367,7 +366,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer builder.addData(dataBuilder.build()); }); - ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToLocalSubscriptionServiceMsg( + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) .build(); return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 89f745260a..1816410ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -33,10 +33,10 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; @Slf4j @Service -public class DefaultLocalSubscriptionService implements LocalSubscriptionService { +public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { private final Set currentPartitions = ConcurrentHashMap.newKeySet(); private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); @@ -68,7 +68,7 @@ public class DefaultLocalSubscriptionService implements LocalSubscriptionService private PartitionService partitionService; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired @Lazy @@ -80,7 +80,7 @@ public class DefaultLocalSubscriptionService implements LocalSubscriptionService @PostConstruct public void initExecutor() { wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index a71155f0ce..b6bee14a32 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; -public interface LocalSubscriptionService { +public interface TbLocalSubscriptionService { void addSubscription(TbSubscription subscription); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ab2803345a..041be297ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -39,10 +39,10 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -73,7 +73,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private TimeseriesService tsService; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired private PartitionService partitionService; @@ -90,7 +90,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 51556602f1..876ea4f2eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -49,7 +49,7 @@ import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import org.thingsboard.server.service.subscription.TbAttributeSubscription; import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; @@ -98,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private LocalSubscriptionService subService; + private TbLocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index fb369f4564..52d748b96b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -23,10 +23,10 @@ import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import java.util.UUID; import java.util.function.Consumer; @@ -43,8 +43,8 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService @Value("${queue.transport.notifications_topic}") private String notificationsTopic; - public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbTransportProducer = tbCoreQueueProvider.getTransportNotificationsMsgProducer(); + public DefaultTbCoreToTransportService(TbQueueProducerProvider tbQueueProducerProvider) { + this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); } @Override diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto deleted file mode 100644 index f0ca4d6d7c..0000000000 --- a/application/src/main/proto/cluster.proto +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package cluster; - -option java_package = "org.thingsboard.server.gen.cluster"; -option java_outer_classname = "ClusterAPIProtos"; - -service ClusterRpcService { - rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} -} - -message ClusterMessage { - MessageType messageType = 1; - MessageMataInfo messageMetaInfo = 2; - ServerAddress serverAddress = 3; - bytes payload = 4; -} - -message ServerAddress { - string host = 1; - int32 port = 2; -} - -message MessageMataInfo { - string payloadMetaInfo = 1; - repeated string tags = 2; -} - -enum MessageType { - - //Cluster control messages - RPC_SESSION_CREATE_REQUEST_MSG = 0; - TO_ALL_NODES_MSG = 1; - RPC_SESSION_TELL_MSG = 2; - RPC_BROADCAST_MSG = 3; - CONNECT_RPC_MESSAGE =4; - - CLUSTER_ACTOR_MESSAGE = 5; - // Messages related to TelemetrySubscriptionService - CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; - CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; - CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; - CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; - CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; - CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; - CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; - - CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; - CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; -} - -// Messages related to CLUSTER_TELEMETRY_MESSAGE - - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c73e623d56..7a80a26713 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -540,6 +540,8 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -548,6 +550,8 @@ queue: print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -561,6 +565,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" transport: + # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 265a552efb..215bd85930 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -150,7 +150,8 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); @@ -266,7 +267,8 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule new TbMsgMetaData(), TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index a4360d51e9..43cdbe6ada 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -143,7 +143,8 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 445465f5df..2634d5af99 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -26,9 +26,9 @@ import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.ArrayList; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 9153e0fc96..dec6f6b50b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; /** @@ -26,9 +27,9 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} + * See {@link PartitionChangeMsg} */ - CLUSTER_EVENT_MSG, + PARTITION_CHANGE_MSG, APP_INIT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 062eb02749..7c7816900b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index a52c237f87..d05948206c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index 56f928a1c8..e9b1a890c9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.common.msg.queue; import lombok.Data; +import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import java.util.Set; + /** * @author Andrew Shvayka */ @Data -public final class ClusterEventMsg implements TbActorMsg { +public final class PartitionChangeMsg implements TbActorMsg { - private final ServerAddress serverAddress; - private final boolean added; + @Getter + private final ServiceKey serviceKey; + @Getter + private final Set partitions; @Override public MsgType getMsgType() { - return MsgType.CLUSTER_EVENT_MSG; + return MsgType.PARTITION_CHANGE_MSG; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java index 3396078124..b68717730b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index a5de33c436..a24a8c4b7c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; public enum ServiceType { TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 1164465ed7..02ad1d0528 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 1a5c98ec80..1b2f984093 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index e7ea35b836..fa9d91d149 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; public interface TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a2e92e112b..34ad3016b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -28,7 +28,7 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 3e70040e2a..4fdd3bdf3c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -23,7 +23,7 @@ import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index fb379c213a..e747b2f99d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceKey; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 92b5723b1e..fdf79d4e1b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -24,6 +24,9 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -156,8 +159,6 @@ public class ConsistentHashPartitionService implements PartitionService { Set tpiList = partitions.stream() .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) .collect(Collectors.toSet()); - // Adding notifications topic for every @TopicPartitionInfo list - tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } }); @@ -184,16 +185,19 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); - services.forEach(serviceInfo -> { - for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + @Override + public Set getAllServiceIds(ServiceType serviceType) { + Set result = new HashSet<>(); + ServiceInfo current = serviceInfoProvider.getServiceInfo(); + if (current.getServiceTypesList().contains(serviceType.name())) { + result.add(current.getServiceId()); + } + for (ServiceInfo serviceInfo : currentOtherServices) { + if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { + result.add(serviceInfo.getServiceId()); } - }); - return currentMap; + } + return result; } @Override @@ -201,17 +205,29 @@ public class ConsistentHashPartitionService implements PartitionService { switch (serviceType) { case TB_CORE: return tbCoreNotificationTopics.computeIfAbsent(serviceId, - id -> buildTopicPartitionInfo(serviceType, serviceId)); + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); case TB_RULE_ENGINE: return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, - id -> buildTopicPartitionInfo(serviceType, serviceId)); + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); default: - return buildTopicPartitionInfo(serviceType, serviceId); + return buildNotificationsTopicPartitionInfo(serviceType, serviceId); } } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null, false); + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + } + }); + return currentMap; + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); } private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index cb7b04df44..2ced989525 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import javax.annotation.PostConstruct; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 13976232e5..05218c68d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 4eb4ca9a69..b22b53f93e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; @@ -38,12 +40,19 @@ public interface PartitionService { */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + /** + * Get all active service ids by service type + * @param serviceType to filter the list of services + * @return list of all active services + */ + Set getAllServiceIds(ServiceType serviceType); + /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. - * @param tbCore + * @param serviceType * @param serviceId * @return */ - TopicPartitionInfo getNotificationsTopic(ServiceType tbCore, String serviceId); + TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 764f646a8a..8e20b35bde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.discovery; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java index 4d02647fd0..2d51bc581a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.AllArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import java.util.Objects; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index ec08d69be7..d43d2a8860 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -24,7 +24,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.io.IOException; import java.time.Duration; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 8f177ecb3f..b749625909 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -27,7 +27,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Properties; import java.util.stream.Collectors; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index c0849d083e..8f226ff2e9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.memory; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index c235f876ce..a2fd7450ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -19,7 +19,7 @@ import lombok.Data; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java index b734562a61..5e1bab5e69 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java @@ -18,6 +18,13 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -25,11 +32,6 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -87,4 +89,24 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul public TbQueueProducer> getTransportApiResponseProducer() { return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java index 3e33e4daa6..17e38b244e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java @@ -39,17 +39,17 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class InMemoryTransportQueueProvider implements TransportQueueProvider { +public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTransportQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryTbTransportQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 990baca5a5..56cf4fb55f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -17,8 +17,11 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; @@ -29,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -38,6 +42,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -45,14 +50,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private TbQueueProducer> tbCoreProducer; - - public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, + public KafkaMonolithQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -79,21 +83,31 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return requestBuilder.build(); } - //TODO 2.5 Singleton + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueProducer> getTbCoreMsgProducer() { - if (tbCoreProducer == null) { - synchronized (this) { - if (tbCoreProducer == null) { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - tbCoreProducer = requestBuilder.build(); - } - } - } - return tbCoreProducer; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override @@ -107,6 +121,17 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); @@ -118,6 +143,17 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 0d6d62c3cb..177f5f3c3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -17,6 +17,14 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -24,11 +32,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -38,6 +42,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -45,14 +50,13 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private TbQueueProducer> tbCoreProducer; - - public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -79,20 +83,31 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { return requestBuilder.build(); } + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueProducer> getTbCoreMsgProducer() { - if (tbCoreProducer == null) { - synchronized (this) { - if (tbCoreProducer == null) { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - tbCoreProducer = requestBuilder.build(); - } - } - } - return tbCoreProducer; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override @@ -106,6 +121,17 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); @@ -125,4 +151,5 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index c75437f38b..36ffd05b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -17,6 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -24,9 +30,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -36,6 +40,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -43,12 +48,13 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -56,6 +62,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; } + @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); @@ -73,6 +80,17 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + + @Override public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); @@ -82,14 +100,34 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider return requestBuilder.build(); } + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("tb-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java index 10cc9c1871..2213889964 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class KafkaTransportQueueProvider implements TransportQueueProvider { +public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -50,12 +50,12 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbTransportQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java new file mode 100644 index 0000000000..c83159242d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { + + private final TbCoreQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + public TbCoreQueueProducerProvider(TbCoreQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index 9fdec01830..bbbef16f65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -44,6 +45,13 @@ public interface TbCoreQueueProvider { */ TbQueueProducer> getRuleEngineMsgProducer(); + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + /** * Used to push messages to other instances of TB Core Service * @@ -51,6 +59,13 @@ public interface TbCoreQueueProvider { */ TbQueueProducer> getTbCoreMsgProducer(); + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** * Used to consume messages by TB Core Service * @@ -58,6 +73,13 @@ public interface TbCoreQueueProvider { */ TbQueueConsumer> getToCoreMsgConsumer(); + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + /** * Used to consume Transport API Calls * diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java new file mode 100644 index 0000000000..34c6ee577f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for providing various Producers to other services. + */ +public interface TbQueueProducerProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java new file mode 100644 index 0000000000..ed7dd43f2a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { + + private final TbRuleEngineQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + + public TbRuleEngineProducerProvider(TbRuleEngineQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java index c4d610cab1..fafa7da9e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. @@ -42,6 +44,13 @@ public interface TbRuleEngineQueueProvider { */ TbQueueProducer> getRuleEngineMsgProducer(); + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + /** * Used to push messages to other instances of TB Core Service * @@ -49,6 +58,13 @@ public interface TbRuleEngineQueueProvider { */ TbQueueProducer> getTbCoreMsgProducer(); + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** * Used to consume messages by TB Core Service * @@ -56,4 +72,11 @@ public interface TbRuleEngineQueueProvider { */ TbQueueConsumer> getToRuleEngineMsgConsumer(); + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java new file mode 100644 index 0000000000..d78f10ef74 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +//TODO 2.5 Maybe remove this service if it is not used. +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { + + private final TbTransportQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + + public TbTransportQueueProducerProvider(TbTransportQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java index 2aa18a62ca..09521692f6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java @@ -25,7 +25,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -public interface TransportQueueProvider { +public interface TbTransportQueueProvider { TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 18c47e542b..87c256174c 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -337,6 +337,12 @@ message LocalSubscriptionServiceMsgProto { TbSubscriptionUpdateProto subUpdate = 1; } +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} /** * Main messages; */ @@ -359,7 +365,14 @@ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; DeviceStateServiceMsgProto deviceStateServiceMsg = 2; SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; - LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 4; + bytes toDeviceActorNotificationMsg = 4; +} + +/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +message ToCoreNotificationMsg { + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; + bytes componentLifecycleMsg = 3; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -369,6 +382,11 @@ message ToRuleEngineMsg { bytes tbMsg = 3; } +message ToRuleEngineNotificationMsg { + bytes componentLifecycleMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; +} + /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { DeviceActorToTransportMsg toDeviceSessionMsg = 1; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 9323cfa7a0..fd4c8e2d95 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,14 +20,8 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; -import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 42d3a534a9..9b8c23bf8b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,7 +17,6 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -44,9 +43,9 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TransportQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbTransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -54,18 +53,15 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -95,7 +91,7 @@ public class DefaultTransportService implements TransportService { private int notificationsPollDuration; private final Gson gson = new Gson(); - private final TransportQueueProvider queueProvider; + private final TbTransportQueueProvider queueProvider; private final PartitionService partitionService; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; @@ -114,7 +110,7 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TransportQueueProvider queueProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueProvider queueProvider, PartitionService partitionService) { this.queueProvider = queueProvider; this.partitionService = partitionService; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 34bfe4d3f4..7cab689d13 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -28,11 +29,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { + private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originHost; - private final int originPort; + private final String originServiceId; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 69ce133c5d..5b76ecf624 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -23,8 +23,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReply(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body); - void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 26f7d14f8b..85fc75139b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} + default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index cd566f8dd2..93114c28e0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { + public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { updateGeneratorState(ctx); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 9444b52210..0c5c7d188e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -58,7 +58,7 @@ public class TbSendRPCReplyNode implements TbNode { } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 9aa94905e6..e4eeef2e5f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -38,7 +36,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -86,10 +83,8 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originHost"); - String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; - tmp = msg.getMetaData().getValue("originPort"); - int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; + tmp = msg.getMetaData().getValue("originServiceId"); + String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -106,16 +101,16 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) + .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originHost(originHost) - .originPort(originPort) + .originServiceId(originServiceId) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); ctx.tellNext(next, TbRelationTypes.SUCCESS); From 6fe25a2ee29ee68e78999050b1ba3265a1466aa7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Sat, 28 Mar 2020 01:42:17 +0200 Subject: [PATCH 024/102] Refactoring to support TB-Rule-Engine service --- .../server/actors/ActorSystemContext.java | 26 +-- .../device/DeviceActorMessageProcessor.java | 23 +-- .../actors/ruleChain/RuleChainActor.java | 2 +- .../actors/ruleChain/RuleNodeActor.java | 2 + .../ThingsboardSecurityConfiguration.java | 1 + .../server/config/WebSocketConfiguration.java | 5 +- .../server/controller/AdminController.java | 14 +- .../server/controller/AlarmController.java | 2 + .../server/controller/AssetController.java | 2 + .../server/controller/AuditLogController.java | 2 + .../server/controller/AuthController.java | 2 + .../server/controller/BaseController.java | 3 + .../ComponentDescriptorController.java | 2 + .../server/controller/CustomerController.java | 2 + .../controller/DashboardController.java | 3 +- .../server/controller/DeviceController.java | 2 + .../controller/EntityRelationController.java | 4 +- .../controller/EntityViewController.java | 2 + .../server/controller/EventController.java | 2 + .../server/controller/RpcController.java | 3 +- .../controller/RuleChainController.java | 2 + .../controller/TelemetryController.java | 2 + .../server/controller/TenantController.java | 2 + .../server/controller/UserController.java | 2 + .../controller/WidgetTypeController.java | 2 + .../controller/WidgetsBundleController.java | 2 + .../controller/plugin/TbWebSocketHandler.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 155 ++++++------------ .../DefaultTbRuleEngineConsumerService.java | 154 +++++------------ .../processing/AbstractConsumerService.java | 150 +++++++++++++++++ .../rpc/DefaultTbCoreDeviceRpcService.java | 5 +- .../rpc/DefaultTbRuleEngineRpcService.java | 7 +- .../auth/rest/RestAuthenticationProvider.java | 2 + ...RestAwareAuthenticationFailureHandler.java | 1 + ...RestAwareAuthenticationSuccessHandler.java | 1 + .../device/DefaultDeviceAuthService.java | 13 +- .../security/model/token/JwtTokenFactory.java | 1 + .../permission/AccessControlService.java | 2 - .../DefaultAccessControlService.java | 1 + .../system/DefaultSystemSecurityService.java | 1 + .../DefaultDeviceSessionCacheService.java | 2 + .../state/DefaultDeviceStateService.java | 18 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultSubscriptionManagerService.java | 2 + .../DefaultTbLocalSubscriptionService.java | 2 + .../DefaultTelemetrySubscriptionService.java | 47 ++++-- .../DefaultTelemetryWebSocketService.java | 2 + .../BaseRuleChainTransactionService.java | 10 +- .../DefaultTbCoreToTransportService.java | 3 +- .../transport/DefaultTransportApiService.java | 2 + ...ce.java => TbCoreTransportApiService.java} | 9 +- .../service/update/DefaultUpdateService.java | 2 + .../src/main/resources/thingsboard.yml | 11 +- .../server/common/msg/queue/ServiceKey.java | 2 + .../common/msg/queue/TopicPartitionInfo.java | 2 + .../provider/KafkaMonolithQueueProvider.java | 6 +- .../provider/KafkaTbCoreQueueProvider.java | 6 +- .../KafkaTbRuleEngineQueueProvider.java | 6 +- .../queue/provider/TbCoreQueueProvider.java | 8 +- .../server/queue/util/TbCoreComponent.java | 15 +- .../server/queue/util/TbKafkaQueue.java | 19 +-- .../queue/util/TbMonolithComponent.java | 19 +-- .../queue/util/TbMonolithOrCoreComponent.java | 22 +++ .../util/TbMonolithOrRuleEngineComponent.java | 22 +++ .../queue/util/TbRuleEngineComponent.java | 22 +++ .../transport/mqtt/MqttTransportHandler.java | 16 +- .../service/DefaultTransportService.java | 9 +- .../session/DeviceAwareSessionContext.java | 8 +- 68 files changed, 526 insertions(+), 380 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java rename application/src/main/java/org/thingsboard/server/service/transport/{RemoteTransportApiService.java => TbCoreTransportApiService.java} (93%) rename application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java (60%) rename application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java (53%) rename application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java (53%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 03b65e7ccc..bd7c25ddc9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -70,7 +70,6 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; @@ -125,10 +124,6 @@ public class ActorSystemContext { @Getter private DataDecodingEncodingService encodingService; - @Autowired - @Getter - private DeviceAuthService deviceAuthService; - @Autowired @Getter private DeviceService deviceService; @@ -212,10 +207,6 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; - @Autowired - @Getter - private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; - @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -232,37 +223,34 @@ public class ActorSystemContext { @Getter private MailService mailService; - @Autowired + //TODO: separate context for TbCore and TbRuleEngine + @Autowired(required = false) @Getter private DeviceStateService deviceStateService; - @Autowired + @Autowired(required = false) @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Lazy - @Autowired + @Autowired(required = false) @Getter private TbCoreToTransportService tbCoreToTransportService; /** * The following Service will be null if we operate in tb-core mode */ - @Lazy - @Getter @Autowired(required = false) + @Getter private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; /** * The following Service will be null if we operate in tb-rule-engine mode */ - @Lazy - @Getter @Autowired(required = false) + @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; - @Lazy - @Autowired + @Autowired(required = false) @Getter private RuleChainTransactionService ruleChainTransactionService; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index aff6200199..307797f864 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -212,8 +212,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - //TODO 2.5 - boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { @@ -225,37 +223,18 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } -// if (msg.hasPostAttributes()) { -// handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); -// reportDeviceActivity = true; -// } -// if (msg.hasPostTelemetry()) { -// handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); -// reportDeviceActivity = true; -// } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } -// if (msg.hasToServerRPCCallRequest()) { -// handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); -// reportDeviceActivity = true; -// } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - if (reportDeviceActivity) { - reportLogicalDeviceActivity(); - } callback.onSuccess(); } - private void reportLogicalDeviceActivity() { - systemContext.getDeviceStateService().onDeviceActivity(deviceId); - } - private void reportSessionOpen() { systemContext.getDeviceStateService().onDeviceConnect(deviceId); } @@ -469,6 +448,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (sessions.size() == 1) { reportSessionOpen(); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -496,6 +476,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 1b9923139a..015e7a008b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -53,7 +53,7 @@ public class RuleChainActor extends ComponentActor implements TbCoreConsumerService { @Value("${queue.core.poll_interval}") private long pollDuration; @@ -72,56 +73,33 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { @Value("${queue.core.stats.enabled:false}") private boolean statsEnabled; - private final ActorSystemContext actorContext; private final DeviceStateService stateService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; - private final DataDecodingEncodingService encodingService; - private final TbQueueConsumer> mainConsumer; - private final TbQueueConsumer> nfConsumer; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - private volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, TbCoreDeviceRpcService tbCoreDeviceRpcService) { - this.mainConsumer = tbCoreQueueProvider.getToCoreMsgConsumer(); - this.nfConsumer = tbCoreQueueProvider.getToCoreNotificationsMsgConsumer(); - this.actorContext = actorContext; + super(actorContext, encodingService, + tbCoreQueueProvider.getToCoreMsgConsumer(), tbCoreQueueProvider.getToCoreNotificationsMsgConsumer()); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; - this.encodingService = encodingService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-notifications-consumer")); + super.init("tb-core-consumer", "tb-core-notifications-consumer"); } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_CORE) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - log.info("Subscribing to notifications: {}", mainConsumer.getTopic()); - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumer(); - } - - private void launchMainConsumer() { + protected void launchMainConsumer() { + log.info("Launching main consumer"); mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -134,6 +112,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { + log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); @@ -177,57 +156,38 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } - private void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); - try { - ToCoreNotificationMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreMsg.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); - } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process notification: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - log.warn("Failed to obtain notifications from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new notifications", e2); - } - } + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_CORE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + ToCoreNotificationMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); + } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); } - log.info("Tb Core Notifications Consumer stopped."); - }); + callback.onSuccess(); + } } private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbMsgCallback callback) { @@ -245,23 +205,6 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - @PreDestroy - public void destroy() { - stopped = true; - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } - private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { if (msg.hasSubUpdate()) { localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index b9d2b98d04..835da15ab7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -19,49 +19,42 @@ import akka.actor.ActorRef; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@TbMonolithOrRuleEngineComponent @Slf4j -public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { @Value("${queue.rule_engine.poll_interval}") private long pollDuration; @@ -70,96 +63,24 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @Value("${queue.rule_engine.stats.enabled:false}") private boolean statsEnabled; - private final ActorSystemContext actorContext; - private final TbQueueConsumer> mainConsumer; - private final TbQueueConsumer> nfConsumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; - private final DataDecodingEncodingService encodingService; - private volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; - private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext, - DataDecodingEncodingService encodingService) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, + ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { + super(actorContext, encodingService, + tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(), tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer()); this.factory = factory; - this.mainConsumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); - this.nfConsumer = tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer(); - this.actorContext = actorContext; - this.encodingService = encodingService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-notifications-consumer")); + super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); this.factory.newInstance(); } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumer(); - } - - private void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); - try { - ToRuleEngineNotificationMsg toRuleEngineMsg = msg.getValue(); - if (toRuleEngineMsg.getComponentLifecycleMsg() != null && !toRuleEngineMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toRuleEngineMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } else { - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process message: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - private void launchMainConsumer() { + protected void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -184,6 +105,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { + log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); try { ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); @@ -217,6 +139,36 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + ToRuleEngineNotificationMsg nfMsg = msg.getValue(); + if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } + private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); @@ -233,20 +185,4 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } } - @PreDestroy - public void destroy() { - stopped = true; - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java new file mode 100644 index 0000000000..e99770faf8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.MsgPackCallback; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractConsumerService implements ApplicationListener { + + protected volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; + protected volatile boolean stopped = false; + + protected final ActorSystemContext actorContext; + protected final DataDecodingEncodingService encodingService; + protected final TbQueueConsumer> mainConsumer; + protected final TbQueueConsumer> nfConsumer; + + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> mainConsumer, TbQueueConsumer> nfConsumer) { + this.actorContext = actorContext; + this.encodingService = encodingService; + this.mainConsumer = mainConsumer; + this.nfConsumer = nfConsumer; + } + + public void init(String mainConsumerThreadName, String nfConsumerThreadName) { + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceKey().getServiceType() == getServiceType()) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + protected abstract ServiceType getServiceType(); + + protected abstract void launchMainConsumer(); + + protected abstract long getNotificationPollDuration(); + + protected abstract long getNotificationPackProcessingTimeout(); + + protected void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + log.info("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + handleNotification(id, msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + log.info("Tb Core Notifications Consumer stopped."); + }); + } + + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception; + + @PreDestroy + public void destroy() { + stopped = true; + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index ed55ee11df..cc9cc4dd10 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; import javax.annotation.PostConstruct; @@ -54,7 +55,7 @@ import java.util.function.Consumer; */ @Service @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +@TbMonolithOrCoreComponent public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private static final ObjectMapper json = new ObjectMapper(); @@ -79,7 +80,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { this.actorContext = actorContext; } - @Autowired + @Autowired(required = false) public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { this.tbRuleEngineRpcService = tbRuleEngineRpcService; } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index 6481f1c2b6..f2390b9629 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.rpc; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; @@ -31,6 +29,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.queue.TbClusterService; import javax.annotation.PostConstruct; @@ -45,7 +44,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@TbMonolithOrRuleEngineComponent @Slf4j public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { @@ -67,7 +66,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi this.serviceInfoProvider = serviceInfoProvider; } - @Autowired + @Autowired(required = false) public void setTbCoreRpcService(Optional tbCoreRpcService) { this.tbCoreRpcService = tbCoreRpcService; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index e359befb30..ac5b3625f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -46,6 +47,7 @@ import ua_parser.Client; import java.util.UUID; + @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java index 726ee76a2d..406b078e6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java @@ -20,6 +20,7 @@ import org.springframework.security.core.AuthenticationException; import org.springframework.security.web.authentication.AuthenticationFailureHandler; import org.springframework.stereotype.Component; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index aa55818084..80cc8ec6e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -23,6 +23,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.web.WebAttributes; import org.springframework.security.web.authentication.AuthenticationSuccessHandler; import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtToken; diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c58a664790..f686cc2037 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -24,16 +24,21 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - @Autowired - DeviceService deviceService; + private final DeviceService deviceService; - @Autowired - DeviceCredentialsService deviceCredentialsService; + private final DeviceCredentialsService deviceCredentialsService; + + public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { + this.deviceService = deviceService; + this.deviceCredentialsService = deviceCredentialsService; + } @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index ff22c00154..f8907b3188 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 40d19da764..5412ed6def 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,11 +15,9 @@ */ package org.thingsboard.server.service.security.permission; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index 9777af7b8e..be99cc1aae 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 11857f25b6..ca6d1f531d 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserServiceImpl; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 952c63c9b9..35c940ef86 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,6 +21,7 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import java.util.Collections; import java.util.UUID; @@ -31,6 +32,7 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 91a49aae1e..986c2cfae7 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -60,6 +60,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -89,6 +90,7 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceStateService implements DeviceStateService { @@ -109,7 +111,8 @@ public class DefaultDeviceStateService implements DeviceStateService { private final TimeseriesService tsService; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; - private final TelemetrySubscriptionService tsSubService; + + private TelemetrySubscriptionService tsSubService; @Value("${state.defaultInactivityTimeoutInSec}") @Getter @@ -137,13 +140,17 @@ public class DefaultDeviceStateService implements DeviceStateService { public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, PartitionService partitionService, TelemetrySubscriptionService tsSubService) { + TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.tenantService = tenantService; this.deviceService = deviceService; this.attributesService = attributesService; this.tsService = tsService; this.producerProvider = producerProvider; this.partitionService = partitionService; + } + + @Autowired + public void setTsSubService(TelemetrySubscriptionService tsSubService) { this.tsSubService = tsSubService; } @@ -188,9 +195,8 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onDeviceActivity(DeviceId deviceId) { - deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { + deviceLastReportedActivity.put(deviceId, lastReportedActivity); long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); @@ -494,7 +500,7 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) - , null, null, null); + , null, null, TbMsgCallback.EMPTY); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 430dd77a36..c7aadda963 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -35,7 +35,7 @@ public interface DeviceStateService extends ApplicationListener currentPartitions = ConcurrentHashMap.newKeySet(); - @Autowired - private AttributesService attrService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - private TbQueueProducerProvider producerProvider; - - @Autowired - private PartitionService partitionService; - - @Autowired - private SubscriptionManagerService subscriptionManagerService; + private final AttributesService attrService; + private final TimeseriesService tsService; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private Optional subscriptionManagerService; private TbQueueProducer> toCoreProducer; private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; + public DefaultTelemetrySubscriptionService(AttributesService attrService, + TimeseriesService tsService, + TbQueueProducerProvider producerProvider, + PartitionService partitionService) { + this.attrService = attrService; + this.tsService = tsService; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + } + + @Autowired(required = false) + public void setSubscriptionManagerService(Optional subscriptionManagerService) { + this.subscriptionManagerService = subscriptionManagerService; + } + @PostConstruct public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); @@ -158,7 +165,11 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + } else { + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); @@ -168,7 +179,11 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + } else { + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 876ea4f2eb..b4c15ee8cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,6 +43,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; @@ -83,6 +84,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index b9424d49d9..b402f88896 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -24,6 +24,8 @@ import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import javax.annotation.PostConstruct; @@ -44,11 +46,11 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; @Service +@TbMonolithOrRuleEngineComponent @Slf4j public class BaseRuleChainTransactionService implements RuleChainTransactionService { - @Autowired - private DbCallbackExecutorService callbackExecutor; + private final DbCallbackExecutorService callbackExecutor; @Value("${actors.rule.transaction.queue_size}") private int finalQueueSize; @@ -61,6 +63,10 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ private ExecutorService timeoutExecutor; + public BaseRuleChainTransactionService(DbCallbackExecutorService callbackExecutor) { + this.callbackExecutor = callbackExecutor; + } + @PostConstruct public void init() { timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 52d748b96b..ab257f00f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import java.util.UUID; import java.util.function.Consumer; @@ -35,7 +36,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Slf4j @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +@TbMonolithOrCoreComponent public class DefaultTbCoreToTransportService implements TbCoreToTransportService { private final TbQueueProducer> tbTransportProducer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 42ee4c4619..dbff5bda86 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -52,6 +53,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service +@TbMonolithOrCoreComponent public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 4c9c801e83..446f7ba81a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -28,6 +28,8 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,9 +40,8 @@ import java.util.concurrent.*; */ @Slf4j @Service -//TODO 2.5: This Confitional annotation should be removed, and Service renamed to something meaningful -//@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteTransportApiService { +@TbMonolithOrCoreComponent +public class TbCoreTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; private final TransportApiService transportApiService; @@ -58,7 +59,7 @@ public class RemoteTransportApiService { private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; - public RemoteTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { + public TbCoreTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { this.tbCoreQueueProvider = tbCoreQueueProvider; this.transportApiService = transportApiService; } diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 1b8173c424..8844c1fe5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,6 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7a80a26713..dd9e6ad3ec 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,8 +205,6 @@ sql: # Actor system parameters actors: - cluster: - grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -370,7 +368,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" # Audit log parameters audit-log: @@ -540,8 +538,6 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -550,8 +546,6 @@ queue: print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -573,4 +567,5 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java index b68717730b..aa6eb27323 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java @@ -16,10 +16,12 @@ package org.thingsboard.server.common.msg.queue; import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; +@ToString public class ServiceKey { @Getter private final ServiceType serviceType; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 02ad1d0528..bf0fffe404 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -17,11 +17,13 @@ package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; +@ToString public class TopicPartitionInfo { private final String topic; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 56cf4fb55f..3566b29478 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,9 +36,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbKafkaQueue; +import org.thingsboard.server.queue.util.TbMonolithComponent; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +@TbMonolithComponent +@TbKafkaQueue public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 177f5f3c3d..12aa66838e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,9 +36,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.queue.util.TbKafkaQueue; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") +@TbCoreComponent +@TbKafkaQueue public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index 36ffd05b48..f295f61d6a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -35,9 +34,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbKafkaQueue; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +@TbKafkaQueue +@TbRuleEngineComponent public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index bbbef16f65..e74c07152b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -50,7 +50,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service @@ -64,7 +64,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> getTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service @@ -78,7 +78,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + TbQueueConsumer> getToCoreNotificationsMsgConsumer(); /** * Used to consume Transport API Calls diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index 50b940ae48..244aae5394 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -13,17 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.util; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToTransportMsgEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(ToTransportMsg value) { - return value.toByteArray(); - } +@ConditionalOnExpression("'${service.type:null}'=='tb-core'") +public @interface TbCoreComponent { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java index 71739c0439..8d4af73492 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java @@ -13,21 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.util; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { - - @Override - public ToRuleEngineMsg decode(TbQueueMsg msg) throws IOException { - return ToRuleEngineMsg.parseFrom(msg.getData()); - } +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public @interface TbKafkaQueue { } diff --git a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java index f4b14144fc..c6dd9ebfff 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java @@ -13,21 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.executors; +package org.thingsboard.server.queue.util; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.AbstractListeningExecutor; - -@Component -public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { - - @Value("${actors.cluster.grpc_callback_thread_pool_size}") - private int grpcCallbackExecutorThreadPoolSize; - - @Override - protected int getThreadPollSize() { - return grpcCallbackExecutorThreadPoolSize; - } +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public @interface TbMonolithComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java new file mode 100644 index 0000000000..a0165e3386 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public @interface TbMonolithOrCoreComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java new file mode 100644 index 0000000000..83e8a2d87a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +public @interface TbMonolithOrRuleEngineComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java new file mode 100644 index 0000000000..9e79232906 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public @interface TbRuleEngineComponent { +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 0d96f94252..fb509e91af 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -410,13 +410,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); - transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - } + doDisconnect(); } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -485,9 +479,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { + doDisconnect(); + } + + private void doDisconnect() { if (deviceSessionCtx.isConnected()) { transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + deviceSessionCtx.setDisconnected(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9b8c23bf8b..3fdf19c741 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -92,6 +93,7 @@ public class DefaultTransportService implements TransportService { private final Gson gson = new Gson(); private final TbTransportQueueProvider queueProvider; + private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; @@ -110,8 +112,9 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueProvider queueProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueProvider queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.queueProvider = queueProvider; + this.producerProvider = producerProvider; this.partitionService = partitionService; } @@ -126,8 +129,8 @@ public class DefaultTransportService implements TransportService { this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); - ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index 7c590ec925..c312b3a4d9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,6 +35,7 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; + private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -42,10 +43,15 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; + this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return deviceInfo != null; + return connected; + } + + public void setDisconnected() { + this.connected = false; } } From c4269023dddd19c3b9c9f6002eabf51e350ec980 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Sat, 28 Mar 2020 02:30:37 +0200 Subject: [PATCH 025/102] Fixed Conditional annotations --- .../ThingsboardSecurityConfiguration.java | 1 - .../server/config/WebSocketConfiguration.java | 4 ++-- .../server/controller/AdminController.java | 4 ++-- .../server/controller/AlarmController.java | 4 ++-- .../server/controller/AssetController.java | 6 ++--- .../server/controller/AuditLogController.java | 4 ++-- .../server/controller/AuthController.java | 4 ++-- .../server/controller/BaseController.java | 5 ++--- .../ComponentDescriptorController.java | 4 ++-- .../server/controller/CustomerController.java | 4 ++-- .../controller/DashboardController.java | 4 ++-- .../server/controller/DeviceController.java | 4 ++-- .../controller/EntityRelationController.java | 4 ++-- .../controller/EntityViewController.java | 4 ++-- .../server/controller/EventController.java | 4 ++-- .../server/controller/RpcController.java | 4 ++-- .../controller/RuleChainController.java | 4 ++-- .../controller/TelemetryController.java | 4 ++-- .../server/controller/TenantController.java | 4 ++-- .../server/controller/UserController.java | 4 ++-- .../controller/WidgetTypeController.java | 4 ++-- .../controller/WidgetsBundleController.java | 5 ++--- .../controller/plugin/TbWebSocketHandler.java | 4 ++-- .../queue/DefaultTbCoreConsumerService.java | 21 +++++++++--------- .../DefaultTbRuleEngineConsumerService.java | 17 ++++++++------ .../processing/AbstractConsumerService.java | 20 +++++++---------- .../rpc/DefaultTbCoreDeviceRpcService.java | 6 ++--- .../rpc/DefaultTbRuleEngineRpcService.java | 4 ++-- .../auth/rest/RestAuthenticationProvider.java | 1 - ...RestAwareAuthenticationFailureHandler.java | 1 - ...RestAwareAuthenticationSuccessHandler.java | 1 - .../device/DefaultDeviceAuthService.java | 5 ++--- .../security/model/token/JwtTokenFactory.java | 1 - .../DefaultAccessControlService.java | 8 ------- .../system/DefaultSystemSecurityService.java | 1 - .../DefaultDeviceSessionCacheService.java | 5 ++--- .../state/DefaultDeviceStateService.java | 6 ++--- .../DefaultSubscriptionManagerService.java | 4 ++-- .../DefaultTbLocalSubscriptionService.java | 4 ++-- .../DefaultTelemetryWebSocketService.java | 4 ++-- .../BaseRuleChainTransactionService.java | 7 ++---- .../DefaultTbCoreToTransportService.java | 5 ++--- .../transport/DefaultTransportApiService.java | 4 ++-- .../transport/TbCoreTransportApiService.java | 3 +-- .../service/update/DefaultUpdateService.java | 4 ++-- application/src/main/resources/logback.xml | 1 + .../provider/KafkaMonolithQueueProvider.java | 6 ++--- .../provider/KafkaTbCoreQueueProvider.java | 6 ++--- .../KafkaTbRuleEngineQueueProvider.java | 6 ++--- .../server/queue/util/TbCoreComponent.java | 2 +- .../server/queue/util/TbKafkaQueue.java | 22 ------------------- .../queue/util/TbMonolithComponent.java | 22 ------------------- .../queue/util/TbMonolithOrCoreComponent.java | 22 ------------------- .../util/TbMonolithOrRuleEngineComponent.java | 22 ------------------- .../queue/util/TbRuleEngineComponent.java | 2 +- .../service/DefaultTransportService.java | 12 +++++----- 56 files changed, 113 insertions(+), 235 deletions(-) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 0921ba91e6..526f0b73c8 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -40,7 +40,6 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenAuthenticationProvider; diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 11cff0f2ac..e57695da69 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,13 +32,13 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration -@TbMonolithOrCoreComponent +@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 5603794d8b..8873d82366 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -30,14 +30,14 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 1412f29daf..c6f3fc3418 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -41,12 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 70164f8f20..03c5462446 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -32,17 +32,15 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -52,7 +50,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 314d2efebd..c99ba8a018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -40,7 +40,7 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 99b36296d4..6722254fd5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -57,7 +57,7 @@ import java.net.URI; import java.net.URISyntaxException; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index f9c3d53ef7..aead345a27 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -92,8 +92,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; -import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; @@ -114,7 +113,7 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j -@TbMonolithOrCoreComponent +@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index 5cc24c55b0..d75da8b34c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,14 +25,14 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 6d2c5fedb4..18fcfc2ab5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,12 +36,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index f0e2ffa176..68f18c9081 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -48,7 +48,7 @@ import java.util.HashSet; import java.util.Set; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index b8a02b281a..2bfb1ed7e8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -63,7 +63,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index e3ef8f4edb..484fa02066 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,14 +33,14 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 1a48d4717b..ebe89bbd10 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -65,7 +65,7 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index 3943bd429f..500549c09a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,11 +31,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index 3dc8db6c0d..731eeb4599 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; @@ -60,7 +60,7 @@ import java.util.UUID; * Created by ashvayka on 22.03.18. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index f91d5540b6..364d348337 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -55,7 +55,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -69,7 +69,7 @@ import java.util.stream.Collectors; @Slf4j @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 31dadafd95..f97efffa92 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -70,7 +70,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -98,7 +98,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 11404cfccb..abd6bf6804 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,13 +34,13 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 881a76325a..a37d9c29b6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -56,7 +56,7 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index ca29acdc1c..debe49b018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,14 +31,14 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index 1520683d34..3d5cd22400 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,15 +32,14 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index b5d94399c1..f7d956e146 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -53,7 +53,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a9f0633eeb..a3613836f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; @@ -56,13 +56,12 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { @@ -78,7 +77,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { while (!stopped) { try { @@ -144,15 +141,17 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { @@ -128,14 +128,17 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } mainConsumer.commit(); } catch (Exception e) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + if (!stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } + log.info("TB Rule Engine Consumer stopped."); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index e99770faf8..0a3ae8f251 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -117,15 +117,17 @@ public abstract class AbstractConsumerService> tbTransportProducer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index dbff5bda86..fdb8c86f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -41,7 +41,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -53,7 +53,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -@TbMonolithOrCoreComponent +@TbCoreComponent public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 446f7ba81a..7a5735b843 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -40,7 +39,7 @@ import java.util.concurrent.*; */ @Slf4j @Service -@TbMonolithOrCoreComponent +@TbCoreComponent public class TbCoreTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 8844c1fe5b..4b7f0f6366 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -39,7 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 3d47db432f..e5de845fbc 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,6 +25,7 @@ + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 3566b29478..56cf4fb55f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -36,12 +37,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbKafkaQueue; -import org.thingsboard.server.queue.util.TbMonolithComponent; @Component -@TbMonolithComponent -@TbKafkaQueue +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 12aa66838e..177f5f3c3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -36,12 +37,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.queue.util.TbKafkaQueue; @Component -@TbCoreComponent -@TbKafkaQueue +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index f295f61d6a..36ffd05b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -34,12 +35,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbKafkaQueue; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; @Component -@TbKafkaQueue -@TbRuleEngineComponent +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index 244aae5394..c42b9b9f7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -17,6 +17,6 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -@ConditionalOnExpression("'${service.type:null}'=='tb-core'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public @interface TbCoreComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java deleted file mode 100644 index 8d4af73492..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") -public @interface TbKafkaQueue { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java deleted file mode 100644 index c6dd9ebfff..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith'") -public @interface TbMonolithComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java deleted file mode 100644 index a0165e3386..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public @interface TbMonolithOrCoreComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java deleted file mode 100644 index 83e8a2d87a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") -public @interface TbMonolithOrRuleEngineComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java index 9e79232906..855bebb19e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -17,6 +17,6 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") public @interface TbRuleEngineComponent { } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3fdf19c741..3891b197a6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -150,11 +150,13 @@ public class DefaultTransportService implements TransportService { }); transportNotificationsConsumer.commit(); } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } From 2553cf6b6fcfb1ccfa8f8d0a5cfbfe1466990f8c Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 30 Mar 2020 19:13:34 +0300 Subject: [PATCH 026/102] created Aws Sqs Queue (#2534) * created Aws Sqs Queue * improvement AwsSqs providers * revert package-lock.json * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * aws improvements * aws improvements * aws improvements * added visibility timeout to aws queue --- .../server/controller/RpcController.java | 1 - .../processing/AbstractConsumerService.java | 9 + .../src/main/resources/thingsboard.yml | 8 +- common/queue/pom.xml | 5 + .../server/queue/TbQueueMsgDecoder.java | 24 ++ .../server/queue/TbQueueProducer.java | 1 + .../common/DefaultTbQueueRequestTemplate.java | 11 + .../DefaultTbQueueResponseTemplate.java | 12 +- .../queue/kafka/TBKafkaConsumerTemplate.java | 7 +- .../queue/kafka/TBKafkaProducerTemplate.java | 5 + .../queue/memory/InMemoryTbQueueProducer.java | 5 + .../provider/AwsSqsMonolithQueueProvider.java | 127 ++++++++++ .../provider/AwsSqsTbCoreQueueProvider.java | 117 +++++++++ .../AwsSqsTbRuleEngineQueueProvider.java | 97 +++++++ .../AwsSqsTransportQueueProvider.java | 93 +++++++ .../queue/sqs/AwsSqsTbQueueMsgMetadata.java | 28 ++ .../server/queue/sqs/TbAwsSqsAdmin.java | 65 +++++ .../queue/sqs/TbAwsSqsConsumerTemplate.java | 239 ++++++++++++++++++ .../server/queue/sqs/TbAwsSqsMsg.java | 38 +++ .../queue/sqs/TbAwsSqsProducerTemplate.java | 127 ++++++++++ .../server/queue/sqs/TbAwsSqsSettings.java | 44 ++++ .../service/DefaultTransportService.java | 7 + pom.xml | 6 + rule-engine/rule-engine-components/pom.xml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 6 +- 25 files changed, 1074 insertions(+), 10 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index 731eeb4599..b88c3f3695 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -88,7 +88,6 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } - private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0a3ae8f251..781762a352 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -136,6 +136,15 @@ public abstract class AbstractConsumerServiceorg.apache.kafka kafka-clients + + com.amazonaws + aws-java-sdk-sqs + + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java new file mode 100644 index 0000000000..4cb38f6685 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.thingsboard.server.queue.TbQueueMsg; + +public interface TbQueueMsgDecoder { + + T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index fa9d91d149..ef308291a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -25,4 +25,5 @@ public interface TbQueueProducer { void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); + void stop(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 34ad3016b0..40ca2219dd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -92,6 +92,8 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); if (responses.size() > 0) { log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } else { + continue; } responses.forEach(response -> { log.trace("Received response to Kafka Template request: {}", response); @@ -145,6 +147,15 @@ public class DefaultTbQueueRequestTemplate requests = requestTemplate.poll(pollInterval); + if (requests.isEmpty()) { + continue; + } + requests.forEach(request -> { long currentTime = System.currentTimeMillis(); long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); @@ -147,6 +151,12 @@ public class DefaultTbQueueResponseTemplate implements TbQueueCon } else { if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(this::createTopicIfNotExists); + topicNames.forEach(admin::createTopicIfNotExists); consumer.subscribe(topicNames); subscribed = true; } @@ -130,7 +130,4 @@ public class TBKafkaConsumerTemplate implements TbQueueCon return decoder.decode(new KafkaTbQueueMsg(record)); } - private void createTopicIfNotExists(String topic) { - admin.createTopicIfNotExists(topic); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index b749625909..2fbfa7d3df 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -84,4 +84,9 @@ public class TBKafkaProducerTemplate implements TbQueuePro } }); } + + @Override + public void stop() { + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index a2fd7450ee..cb1ff939d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -50,4 +50,9 @@ public class InMemoryTbQueueProducer implements TbQueuePro } } } + + @Override + public void stop() { + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java new file mode 100644 index 0000000000..206a71d85e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") +public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + + public AwsSqsMonolithQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java new file mode 100644 index 0000000000..052d3fc0e3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") +public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + + private final TbQueueAdmin admin; + + public AwsSqsTbCoreQueueProvider(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..d9f2cec9f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") +public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + + public AwsSqsTbRuleEngineQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java new file mode 100644 index 0000000000..01360ebed8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public AwsSqsTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = + new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + + TbAwsSqsConsumerTemplate> consumerTemplate = + new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java new file mode 100644 index 0000000000..eb4eaed7ed --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.http.SdkHttpMetadata; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +@Data +@AllArgsConstructor +public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { + + private final SdkHttpMetadata metadata; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java new file mode 100644 index 0000000000..6080baec26 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.CreateQueueRequest; +import com.amazonaws.services.sqs.model.QueueAttributeName; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.HashMap; +import java.util.Map; + +public class TbAwsSqsAdmin implements TbQueueAdmin { + + private final TbAwsSqsSettings sqsSettings; + private final Map attributes = new HashMap<>(); + private final AWSStaticCredentialsProvider credProvider; + + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings) { + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + this.credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + attributes.put("FifoQueue", "true"); + attributes.put("ContentBasedDeduplication", "true"); + attributes.put(QueueAttributeName.VisibilityTimeout.toString(), sqsSettings.getVisibilityTimeout()); + } + + @Override + public void createTopicIfNotExists(String topic) { + AmazonSQS sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + final CreateQueueRequest createQueueRequest = + new CreateQueueRequest(topic.replaceAll("\\.", "_") + ".fifo") + .withAttributes(attributes); + try { + sqsClient.createQueue(createQueueRequest); + } finally { + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java new file mode 100644 index 0000000000..307e05bcec --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.google.common.reflect.TypeToken; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { + + private static final int MAX_NUM_MSGS = 10; + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final AmazonSQS sqsClient; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbAwsSqsSettings sqsSettings; + + private final List pendingMessages = new CopyOnWriteArrayList<>(); + private volatile Set queueUrls; + private volatile Set partitions; + private ListeningExecutorService consumerExecutor; + private volatile boolean subscribed; + private volatile boolean stopped = false; + + public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + + if (sqsClient != null) { + sqsClient.shutdown(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); + subscribed = true; + } + + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .map(msg -> { + try { + return decode(msg); + } catch (IOException e) { + log.error("Failed to decode message: [{}]", msg); + return null; + } + }).filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", topic); + } else { + log.error("Failed to pool messages.", e); + } + } + } + return Collections.emptyList(); + } + + private ListenableFuture> poll(String url, int waitTimeSeconds) { + List>> result = new ArrayList<>(); + + for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { + result.add(consumerExecutor.submit(() -> { + ReceiveMessageRequest request = new ReceiveMessageRequest(); + request + .withWaitTimeSeconds(waitTimeSeconds) + .withMessageAttributeNames("headers") + .withQueueUrl(url) + .withMaxNumberOfMessages(MAX_NUM_MSGS); + return sqsClient.receiveMessage(request).getMessages(); + })); + } + return Futures.transform(Futures.allAsList(result), list -> { + if (!CollectionUtils.isEmpty(list)) { + return list.stream() + .flatMap(messageList -> { + if (!messageList.isEmpty()) { + this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); + return messageList.stream(); + } + return Stream.empty(); + }) + .collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + } + + @Override + public void commit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + + pendingMessages.clear(); + } + + public T decode(Message message) throws InvalidProtocolBufferException { + TbAwsSqsMsg msg = gson.fromJson(message.getBody(), TbAwsSqsMsg.class); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { + }.getType()); + headerMap.forEach(headers::put); + msg.setHeaders(headers); + return decoder.decode(msg); + } + + @Data + private static class AwsSqsMsgWrapper { + private final String url; + private final List messages; + + public AwsSqsMsgWrapper(String url, List messages) { + this.url = url; + this.messages = messages; + } + } + + private String getQueueUrl(String topic) { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java new file mode 100644 index 0000000000..4df7558491 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.google.gson.annotations.Expose; +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbAwsSqsMsg implements TbQueueMsg { + private final UUID key; + private final byte[] data; + + public TbAwsSqsMsg(UUID key, byte[] data) { + this.key = key; + this.data = data; + } + + @Expose(serialize = false, deserialize = false) + private TbQueueMsgHeaders headers; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java new file mode 100644 index 0000000000..6514d12747 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.MessageAttributeValue; +import com.amazonaws.services.sqs.model.SendMessageRequest; +import com.amazonaws.services.sqs.model.SendMessageResult; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; + +@Slf4j +public class TbAwsSqsProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final AmazonSQS sqsClient; + private final Gson gson = new Gson(); + private final Map queueUrlMap = new ConcurrentHashMap<>(); + private final TbQueueAdmin admin; + private ListeningExecutorService producerExecutor; + + public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + SendMessageRequest sendMsgRequest = new SendMessageRequest(); + sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); + sendMsgRequest.withMessageBody(gson.toJson(new TbAwsSqsMsg(msg.getKey(), msg.getData()))); + + Map attributes = new HashMap<>(); + + attributes.put("headers", new MessageAttributeValue() + .withStringValue(gson.toJson(msg.getHeaders().getData())) + .withDataType("String")); + + sendMsgRequest.withMessageAttributes(attributes); + sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(SendMessageResult result) { + if (callback != null) { + callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + + private String getQueueUrl(String topic) { + return queueUrlMap.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java new file mode 100644 index 0000000000..42a2f1e983 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +@Component +@Data +public class TbAwsSqsSettings { + + @Value("${queue.aws_sqs.access_key_id}") + private String accessKeyId; + + @Value("${queue.aws_sqs.secret_access_key}") + private String secretAccessKey; + + @Value("${queue.aws_sqs.region}") + private String region; + + @Value("${queue.aws_sqs.threads_per_topic}") + private int threadsPerTopic; + + @Value("${queue.aws_sqs.visibility_timeout}") + private String visibilityTimeout; +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3891b197a6..3a707d11a7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -138,6 +138,9 @@ public class DefaultTransportService implements TransportService { while (!stopped) { try { List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + if (records.size() == 0) { + continue; + } records.forEach(record -> { try { ToTransportMsg toTransportMsg = record.getValue(); @@ -170,6 +173,10 @@ public class DefaultTransportService implements TransportService { perDeviceLimits.clear(); } stopped = true; + + if (transportNotificationsConsumer != null) { + transportNotificationsConsumer.unsubscribe(); + } if (schedulerExecutor != null) { schedulerExecutor.shutdownNow(); } diff --git a/pom.xml b/pom.xml index 547c97448f..9863982929 100755 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ 2.57 2.7.7 1.23 + 1.11.747 1.5.0 1.4.3 @@ -886,6 +887,11 @@ jts-core ${jts.version} + + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + org.passay passay diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 373e4107f8..bf629a6f10 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,7 +35,7 @@ UTF-8 ${basedir}/../.. - 1.11.323 + 1.11.747 1.83.0 1.16.0 diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index f4a31449e7..6ac9b08777 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or ? + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -71,6 +71,10 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From a6e090ef868fab068c21a65dc76a11c8c240e780 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 31 Mar 2020 16:39:41 +0300 Subject: [PATCH 027/102] Develop/2.5 pubsub (#2566) * created Aws Sqs Queue * improvement AwsSqs providers * created pubsub queue * revert package-lock.json * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Created pubsub queue * aws improvements * aws improvements * aws improvements * added visibility timeout to aws queue * pub sub improvements * pub sub improvements * aws sqs improvements * pub sub improvements * added comment to transport.yml about ack deadline --- .../src/main/resources/thingsboard.yml | 10 +- common/queue/pom.xml | 5 +- .../DefaultTbQueueMsg.java} | 7 +- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../provider/KafkaTbCoreQueueProvider.java | 6 +- .../KafkaTbRuleEngineQueueProvider.java | 10 +- .../provider/PubSubMonolithQueueProvider.java | 137 +++++++++++ .../provider/PubSubTbCoreQueueProvider.java | 113 +++++++++ .../PubSubTbRuleEngineQueueProvider.java | 101 ++++++++ .../PubSubTransportQueueProvider.java | 104 ++++++++ .../server/queue/pubsub/TbPubSubAdmin.java | 157 ++++++++++++ .../pubsub/TbPubSubConsumerTemplate.java | 228 ++++++++++++++++++ .../pubsub/TbPubSubProducerTemplate.java | 135 +++++++++++ .../server/queue/pubsub/TbPubSubSettings.java | 61 +++++ .../queue/sqs/TbAwsSqsConsumerTemplate.java | 3 +- .../queue/sqs/TbAwsSqsProducerTemplate.java | 3 +- pom.xml | 16 +- rule-engine/rule-engine-components/pom.xml | 2 - .../src/main/resources/tb-mqtt-transport.yml | 10 +- 19 files changed, 1078 insertions(+), 32 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/queue/{sqs/TbAwsSqsMsg.java => common/DefaultTbQueueMsg.java} (86%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2fe8f59608..e731bd7bac 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -517,7 +517,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -530,7 +530,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #in seconds + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e201b8995f..0a61eb0cf6 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -56,7 +56,10 @@ com.amazonaws aws-java-sdk-sqs - + + com.google.cloud + google-cloud-pubsub + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java similarity index 86% rename from common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index 4df7558491..0e816ae59b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.sqs; +package org.thingsboard.server.queue.common; import com.google.gson.annotations.Expose; import lombok.Data; @@ -23,16 +23,15 @@ import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; @Data -public class TbAwsSqsMsg implements TbQueueMsg { +public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; - public TbAwsSqsMsg(UUID key, byte[] data) { + public DefaultTbQueueMsg(UUID key, byte[] data) { this.key = key; this.data = data; } @Expose(serialize = false, deserialize = false) private TbQueueMsgHeaders headers; - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 40ca2219dd..6ae9b9a33c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate { - log.trace("Received response to Kafka Template request: {}", response); + log.trace("Received response to Queue Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; if (requestIdHeader == null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 177f5f3c3d..110a98b3b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -30,7 +30,6 @@ import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -48,21 +47,18 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportApiSettings transportApiSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index 36ffd05b48..7f939ee11d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -27,8 +27,6 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -45,22 +43,16 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueRuleEngineSettings ruleEngineSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java new file mode 100644 index 0000000000..2dc9549679 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") +public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private TbQueueProducer> tbCoreProducer; + + public PubSubMonolithQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.admin = new TbPubSubAdmin(pubSubSettings); + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java new file mode 100644 index 0000000000..4c89b35dec --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") +public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + public PubSubTbCoreQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.admin = admin; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..3f707235fa --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") +public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + public PubSubTbRuleEngineQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.admin = admin; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java new file mode 100644 index 0000000000..b1a7efc950 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class PubSubTransportQueueProvider implements TbTransportQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin admin; + + public PubSubTransportQueueProvider(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.pubSubSettings = pubSubSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.admin = new TbPubSubAdmin(pubSubSettings); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java new file mode 100644 index 0000000000..f0af639d4d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -0,0 +1,157 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.pubsub.v1.ListSubscriptionsRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ProjectName; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.Topic; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +public class TbPubSubAdmin implements TbQueueAdmin { + + private final TbPubSubSettings pubSubSettings; + private final SubscriptionAdminSettings subscriptionAdminSettings; + private final TopicAdminSettings topicAdminSettings; + private final Set topicSet = ConcurrentHashMap.newKeySet(); + private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + + public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { + this.pubSubSettings = pubSubSettings; + + try { + topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create TopicAdminSettings"); + throw new RuntimeException("Failed to create TopicAdminSettings."); + } + + try { + subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create SubscriptionAdminSettings"); + throw new RuntimeException("Failed to create SubscriptionAdminSettings."); + } + + try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + topicSet.add(topic.getName()); + } + } catch (IOException e) { + log.error("Failed to get topics.", e); + throw new RuntimeException("Failed to get topics.", e); + } + + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + subscriptionSet.add(subscription.getName()); + } + } catch (IOException e) { + log.error("Failed to get subscriptions.", e); + throw new RuntimeException("Failed to get subscriptions.", e); + } + } + + @Override + public void createTopicIfNotExists(String partition) { + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + + if (topicSet.contains(topicName.toString())) { + createSubscriptionIfNotExists(partition, topicName); + return; + } + + try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; + } + } + + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); + } catch (IOException e) { + log.error("Failed to create topic: [{}].", topicName.toString(), e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); + + if (subscriptionSet.contains(subscriptionName.toString())) { + return; + } + + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; + } + } + + subscriptionAdminClient.createSubscription( + subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } catch (IOException e) { + log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); + throw new RuntimeException("Failed to create subscription.", e); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java new file mode 100644 index 0000000000..4109e72070 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +public class TbPubSubConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbPubSubSettings pubSubSettings; + + private volatile boolean subscribed; + private volatile Set partitions; + private volatile Set subscriptionNames; + private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); + + private ExecutorService consumerExecutor; + private final SubscriberStub subscriber; + private volatile boolean stopped; + + private volatile int messagesPerTopic; + + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.pubSubSettings = pubSubSettings; + this.topic = topic; + this.decoder = decoder; + + try { + SubscriberStubSettings subscriberStubSettings = + SubscriberStubSettings.newBuilder() + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setTransportChannelProvider( + SubscriberStubSettings.defaultGrpcTransportProviderBuilder() + .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) + .build()) + .build(); + + this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); + } catch (IOException e) { + log.error("Failed to create subscriber.", e); + throw new RuntimeException("Failed to create subscriber.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + + if (subscriber != null) { + subscriber.close(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); + subscriptionNames.forEach(admin::createTopicIfNotExists); + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); + messagesPerTopic = pubSubSettings.getMaxMessages()/subscriptionNames.size(); + subscribed = true; + } + List messages; + try { + messages = receiveMessages(); + if (!messages.isEmpty()) { + List result = new ArrayList<>(); + messages.forEach(msg -> { + try { + result.add(decode(msg.getMessage())); + } catch (InvalidProtocolBufferException e) { + log.error("Failed decode record: [{}]", msg); + } + }); + return result; + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); + } + + private List receiveMessages() throws ExecutionException, InterruptedException { + List>> result = subscriptionNames.stream().map(subscriptionId -> { + String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); + PullRequest pullRequest = + PullRequest.newBuilder() + .setMaxMessages(messagesPerTopic) + .setReturnImmediately(false) // return immediately if messages are not available + .setSubscription(subscriptionName) + .build(); + + ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); + + return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { + if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { + List ackIds = new ArrayList<>(); + for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { + ackIds.add(message.getAckId()); + } + AcknowledgeRequest acknowledgeRequest = + AcknowledgeRequest.newBuilder() + .setSubscription(subscriptionName) + .addAllAckIds(ackIds) + .build(); + + acknowledgeRequests.add(acknowledgeRequest); + return pullResponse.getReceivedMessagesList(); + } + return null; + }, consumerExecutor); + + }).collect(Collectors.toList()); + + ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { + if (!CollectionUtils.isEmpty(listMessages)) { + return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + + return transform.get(); + } + + public T decode(PubsubMessage message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + Map headerMap = gson.fromJson(message.getAttributesMap().get("headers"), new TypeToken>() { + }.getType()); + headerMap.forEach(headers::put); + msg.setHeaders(headers); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java new file mode 100644 index 0000000000..fb9bc8b189 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.gson.Gson; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class TbPubSubProducerTemplate implements TbQueueProducer { + + private final Gson gson = new Gson(); + + private final String defaultTopic; + private final TbQueueAdmin admin; + private final TbPubSubSettings pubSubSettings; + + private final Map publisherMap = new ConcurrentHashMap<>(); + + private ExecutorService pubExecutor = Executors.newCachedThreadPool(); + + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { + this.defaultTopic = defaultTopic; + this.admin = admin; + this.pubSubSettings = pubSubSettings; + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); + pubsubMessageBuilder.setData(getMsg(msg)); + pubsubMessageBuilder.putAttributes("headers", gson.toJson(msg.getHeaders().getData())); + + Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); + ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); + + ApiFutures.addCallback(future, new ApiFutureCallback() { + public void onSuccess(String messageId) { + if (callback != null) { + callback.onSuccess(null); + } + } + + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, pubExecutor); + } + + @Override + public void stop() { + publisherMap.forEach((k, v) -> { + if (v != null) { + try { + v.shutdown(); + v.awaitTermination(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to shutdown PubSub client during destroy()", e); + } + } + }); + + if (pubExecutor != null) { + pubExecutor.shutdownNow(); + } + } + + private ByteString getMsg(T msg) { + String json = gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData())); + return ByteString.copyFrom(json.getBytes()); + } + + private Publisher getOrCreatePublisher(String topic) { + if (publisherMap.containsKey(topic)) { + return publisherMap.get(topic); + } else { + try { + admin.createTopicIfNotExists(topic); + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); + Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + publisherMap.put(topic, publisher); + return publisher; + } catch (IOException e) { + log.error("Failed to create topic [{}].", topic, e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java new file mode 100644 index 0000000000..851a2b3245 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.ByteArrayInputStream; +import java.io.IOException; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +@Component +@Data +public class TbPubSubSettings { + + @Value("${queue.pubsub.project_id}") + private String projectId; + + @Value("${queue.pubsub.service_account}") + private String serviceAccount; + + @Value("${queue.pubsub.ack_deadline}") + private int ackDeadline; + + @Value("${queue.pubsub.max_msg_size}") + private int maxMsgSize; + + @Value("${queue.pubsub.max_messages}") + private int maxMessages; + + private CredentialsProvider credentialsProvider; + + @PostConstruct + private void init() throws IOException { + ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( + new ByteArrayInputStream(serviceAccount.getBytes())); + credentialsProvider = FixedCredentialsProvider.create(credentials); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 307e05bcec..b4a2f84b83 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; @@ -212,7 +213,7 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo } public T decode(Message message) throws InvalidProtocolBufferException { - TbAwsSqsMsg msg = gson.fromJson(message.getBody(), TbAwsSqsMsg.class); + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { }.getType()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 6514d12747..a5707c845d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -35,6 +35,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.HashMap; import java.util.Map; @@ -79,7 +80,7 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { SendMessageRequest sendMsgRequest = new SendMessageRequest(); sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new TbAwsSqsMsg(msg.getKey(), msg.getData()))); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); Map attributes = new HashMap<>(); diff --git a/pom.xml b/pom.xml index 9863982929..e385bff9ce 100755 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 1.4.3 4.2.0 3.5.5 - 3.6.1 + 3.9.1 1.22.1 1.16.18 1.1.0 @@ -93,6 +93,7 @@ 2.7.7 1.23 1.11.747 + 1.84.0 1.5.0 1.4.3 @@ -888,10 +889,15 @@ ${jts.version} - com.amazonaws - aws-java-sdk-sqs - ${amazonaws.sqs.version} - + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + + + com.google.cloud + google-cloud-pubsub + ${pubsub.client.version} + org.passay passay diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index bf629a6f10..645419ca41 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -36,7 +36,6 @@ UTF-8 ${basedir}/../.. 1.11.747 - 1.83.0 1.16.0 @@ -99,7 +98,6 @@ com.google.cloud google-cloud-pubsub - ${pubsub.client.version} com.google.api.grpc diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 6ac9b08777..a875d56782 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -75,6 +75,14 @@ queue: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From b39328c989a36711c133de67d753cbdedc2a4467 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 31 Mar 2020 16:47:43 +0300 Subject: [PATCH 028/102] [2.5]fix ConcurrentModificationException (#2560) * fix ConcurrentModificationException * kafka consumer improvements * kafka consumer improvements * refactored kafka consumer * refactored kafka consumer --- .../queue/kafka/TBKafkaConsumerTemplate.java | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 68b9cea7fa..d882153177 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -33,6 +33,8 @@ import java.util.Collections; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -46,6 +48,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final TbKafkaDecoder decoder; private volatile boolean subscribed; private volatile Set partitions; + private final Lock consumerLock; @Getter private final String topic; @@ -71,6 +74,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; + this.consumerLock = new ReentrantLock(); } @Override @@ -94,23 +98,30 @@ public class TBKafkaConsumerTemplate implements TbQueueCon log.debug("Failed to await subscription", e); } } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; + try { + consumerLock.lock(); + + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } + + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + } finally { + consumerLock.unlock(); } } return Collections.emptyList(); @@ -118,12 +129,25 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void commit() { - consumer.commitAsync(); + try { + consumerLock.lock(); + consumer.commitAsync(); + } finally { + consumerLock.unlock(); + } } @Override public void unsubscribe() { - consumer.unsubscribe(); + try { + consumerLock.lock(); + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); + } + } finally { + consumerLock.unlock(); + } } public T decode(ConsumerRecord record) throws IOException { From 638ca0e1d104d8ce2ce744e284a777abb0b61bca Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 18:35:43 +0300 Subject: [PATCH 029/102] Refactoring to multiple queues for rule engine --- .../server/actors/ActorSystemContext.java | 13 +- .../actors/ruleChain/DefaultTbContext.java | 167 ++++++++++++------ .../RuleChainActorMessageProcessor.java | 4 +- .../RuleNodeActorMessageProcessor.java | 2 +- .../actors/service/DefaultActorService.java | 12 +- .../server/actors/tenant/TenantActor.java | 2 +- .../server/controller/BaseController.java | 4 +- .../controller/RuleChainController.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 41 +++-- .../DefaultTbRuleEngineConsumerService.java | 84 +++++++-- .../processing/AbstractConsumerService.java | 37 ++-- ...TbRuleEngineProcessingStrategyFactory.java | 34 ++-- .../rpc/DefaultTbCoreDeviceRpcService.java | 4 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../state/DefaultDeviceStateService.java | 8 +- .../DefaultTbLocalSubscriptionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../transport/TbCoreTransportApiService.java | 12 +- application/src/main/resources/logback.xml | 1 - .../src/main/resources/thingsboard.yml | 46 +++-- ...AbstractRuleEngineFlowIntegrationTest.java | 9 +- ...actRuleEngineLifecycleIntegrationTest.java | 5 +- .../script/RuleNodeJsScriptEngineTest.java | 14 +- .../thingsboard/server/common/msg/TbMsg.java | 30 +++- .../common/msg/queue/PartitionChangeMsg.java | 2 +- .../msg/queue/QueueToRuleEngineMsg.java | 6 + .../server/common/msg/queue/ServiceQueue.java | 62 +++++++ .../{ServiceKey.java => ServiceQueueKey.java} | 19 +- .../server/common/msg/queue/ServiceType.java | 1 + .../MultipleTbQueueTbMsgCallbackWrapper.java | 4 +- .../TbQueueTbMsgCallbackWrapper.java | 4 +- .../discovery/ClusterTopologyChangeEvent.java | 8 +- .../ConsistentHashPartitionService.java | 154 +++++++++------- .../DefaultTbServiceInfoProvider.java | 22 ++- .../queue/discovery/PartitionChangeEvent.java | 13 +- .../queue/discovery/PartitionService.java | 4 +- .../discovery/TbServiceInfoProvider.java | 5 - .../discovery/TopicPartitionInfoKey.java | 8 +- ...r.java => AwsSqsMonolithQueueFactory.java} | 45 ++--- ...der.java => AwsSqsTbCoreQueueFactory.java} | 38 ++-- ...va => AwsSqsTbRuleEngineQueueFactory.java} | 29 +-- ....java => AwsSqsTransportQueueFactory.java} | 22 +-- ...java => InMemoryMonolithQueueFactory.java} | 41 ++--- ...a => InMemoryTbTransportQueueFactory.java} | 26 +-- ...er.java => KafkaMonolithQueueFactory.java} | 50 +++--- ...ider.java => KafkaTbCoreQueueFactory.java} | 36 ++-- ...ava => KafkaTbRuleEngineQueueFactory.java} | 34 ++-- ...java => KafkaTbTransportQueueFactory.java} | 30 ++-- ...r.java => PubSubMonolithQueueFactory.java} | 47 ++--- ...der.java => PubSubTbCoreQueueFactory.java} | 36 ++-- ...va => PubSubTbRuleEngineQueueFactory.java} | 33 ++-- ....java => PubSubTransportQueueFactory.java} | 30 ++-- ...eProvider.java => TbCoreQueueFactory.java} | 20 +-- .../provider/TbCoreQueueProducerProvider.java | 14 +- .../TbRuleEngineProducerProvider.java | 14 +- ...der.java => TbRuleEngineQueueFactory.java} | 19 +- ...ider.java => TbTransportQueueFactory.java} | 10 +- .../TbTransportQueueProducerProvider.java | 8 +- .../{ => settings}/TbQueueCoreSettings.java | 2 +- .../settings/TbQueueRuleEngineSettings.java | 45 +++++ .../TbQueueTransportApiSettings.java | 2 +- .../TbQueueTransportNotificationSettings.java | 2 +- ...leEngineQueueAckStrategyConfiguration.java | 32 ++++ .../TbRuleEngineQueueConfiguration.java} | 15 +- common/queue/src/main/proto/queue.proto | 8 + .../service/DefaultTransportService.java | 18 +- .../rule/engine/api/TbContext.java | 79 ++++++++- .../engine/action/TbAbstractAlarmNode.java | 10 +- .../action/TbAbstractCustomerActionNode.java | 4 +- .../action/TbAbstractRelationActionNode.java | 12 +- .../rule/engine/action/TbMsgCountNode.java | 7 +- .../rule/engine/delay/TbMsgDelayNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 4 +- .../rule/engine/kafka/TbKafkaNode.java | 3 +- .../engine/rest/TbRedisQueueProcessor.java | 2 +- .../TbSynchronizationBeginNode.java | 3 +- .../rule/engine/action/TbAlarmNodeTest.java | 10 +- .../engine/filter/TbJsFilterNodeTest.java | 6 +- .../engine/filter/TbJsSwitchNodeTest.java | 2 +- .../engine/mail/TbMsgToEmailNodeTest.java | 2 +- .../TbGetCustomerAttributeNodeTest.java | 16 +- .../transform/TbChangeOriginatorNodeTest.java | 6 +- .../transform/TbTransformMsgNodeTest.java | 6 +- 83 files changed, 1066 insertions(+), 665 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java rename common/message/src/main/java/org/thingsboard/server/common/msg/queue/{ServiceKey.java => ServiceQueueKey.java} (72%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => common}/MultipleTbQueueTbMsgCallbackWrapper.java (90%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => common}/TbQueueTbMsgCallbackWrapper.java (88%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsMonolithQueueProvider.java => AwsSqsMonolithQueueFactory.java} (76%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTbCoreQueueProvider.java => AwsSqsTbCoreQueueFactory.java} (76%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTbRuleEngineQueueProvider.java => AwsSqsTbRuleEngineQueueFactory.java} (74%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTransportQueueProvider.java => AwsSqsTransportQueueFactory.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryMonolithQueueProvider.java => InMemoryMonolithQueueFactory.java} (70%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryTbTransportQueueProvider.java => InMemoryTbTransportQueueFactory.java} (77%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaMonolithQueueProvider.java => KafkaMonolithQueueFactory.java} (82%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbCoreQueueProvider.java => KafkaTbCoreQueueFactory.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbRuleEngineQueueProvider.java => KafkaTbRuleEngineQueueFactory.java} (80%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbTransportQueueProvider.java => KafkaTbTransportQueueFactory.java} (83%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubMonolithQueueProvider.java => PubSubMonolithQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTbCoreQueueProvider.java => PubSubTbCoreQueueFactory.java} (78%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTbRuleEngineQueueProvider.java => PubSubTbRuleEngineQueueFactory.java} (74%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTransportQueueProvider.java => PubSubTransportQueueFactory.java} (79%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbCoreQueueProvider.java => TbCoreQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbRuleEngineQueueProvider.java => TbRuleEngineQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbTransportQueueProvider.java => TbTransportQueueFactory.java} (78%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueCoreSettings.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueTransportApiSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueTransportNotificationSettings.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java rename common/queue/src/main/java/org/thingsboard/server/queue/{TbQueueRuleEngineSettings.java => settings/TbRuleEngineQueueConfiguration.java} (71%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index bd7c25ddc9..696383b242 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -31,7 +31,6 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -48,7 +47,6 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -413,7 +411,12 @@ public class ActorSystemContext { return partitionService.resolve(serviceType, tenantId, entityId); } - public String getServerAddress() { + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, queueName, tenantId, entityId); + } + + + public String getServiceId() { return serviceInfoProvider.getServiceId(); } @@ -445,7 +448,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -505,7 +508,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("message", "Reached debug mode rate limit!"); if (error != null) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7c831929ca..2f31d1dc35 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -22,12 +22,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -37,18 +36,17 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -63,10 +61,11 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -78,6 +77,7 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ +@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -100,11 +100,6 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } - @Override - public void tellNext(TbMsg msg, String relationType, Throwable th) { - tellNext(msg, Collections.singleton(relationType), th); - } - private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); @@ -118,6 +113,77 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } + @Override + public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + @Override + public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + } + + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)) + .addAllRelationTypes(relationTypes) + .build(); + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void ack(TbMsg tbMsg) { + tbMsg.getCallback().onSuccess(); + } + @Override public boolean isLocalEntity(EntityId entityId) { return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); @@ -135,68 +201,44 @@ class DefaultTbContext implements TbContext { nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); } - @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), null); + return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), null); - } - - @Override - public void sendTbMsgToRuleEngine(TbMsg tbMsg) { - mainCtx.getClusterService().onToRuleEngineMsg(getTenantId(), tbMsg.getOriginator(), tbMsg); + return TbMsg.transformMsg(origMsg, type, originator, metaData, data); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), - getActionMetaData(ruleNodeId), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process customer created msg: " + e); - } + return entityCreatedMsg(customer, customer.getId(), ruleNodeId); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process device created msg: " + e); - } + return entityCreatedMsg(device, device.getId(), ruleNodeId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process asset created msg: " + e); - } + return entityCreatedMsg(asset, asset.getId(), ruleNodeId); } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { + return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + } + + public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); + return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process alarm created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); } } - @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -254,7 +296,7 @@ class DefaultTbContext implements TbContext { } @Override - public String getNodeId() { + public String getServiceId() { return mainCtx.getServiceInfoProvider().getServiceId(); } @@ -362,10 +404,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } - @Override - public String getServerAddress() { - return mainCtx.getServerAddress(); - } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -373,4 +411,29 @@ class DefaultTbContext implements TbContext { return metaData; } + private class SimpleTbQueueCallback implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } else { + log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 50aa7e5dd6..00d9d85b1c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -39,10 +39,10 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.MultipleTbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index 2dd6764407..4cd668cc36 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -40,7 +40,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor implements TbCoreConsumerService { +public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { - @Value("${queue.core.poll_interval}") + @Value("${queue.core.poll-interval}") private long pollDuration; - @Value("${queue.core.pack_processing_timeout}") + @Value("${queue.core.pack-processing-timeout}") private long packProcessingTimeout; @Value("${queue.core.stats.enabled:false}") private boolean statsEnabled; + private final TbQueueConsumer> mainConsumer; private final DeviceStateService stateService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, TbCoreDeviceRpcService tbCoreDeviceRpcService) { - super(actorContext, encodingService, - tbCoreQueueProvider.getToCoreMsgConsumer(), tbCoreQueueProvider.getToCoreNotificationsMsgConsumer()); + super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; @@ -96,8 +99,16 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumersExecutor.submit(() -> { while (!stopped) { try { List> msgs = mainConsumer.poll(pollDuration); @@ -171,7 +182,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbMsgCallback callback) throws Exception { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) { ToCoreNotificationMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); @@ -197,7 +208,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - @Value("${queue.rule_engine.poll_interval}") + @Value("${queue.rule-engine.poll-interval}") private long pollDuration; - @Value("${queue.rule_engine.pack_processing_timeout}") + @Value("${queue.rule-engine.pack-processing-timeout}") private long packProcessingTimeout; - @Value("${queue.rule_engine.stats.enabled:false}") + @Value("${queue.rule-engine.stats.enabled:false}") private boolean statsEnabled; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; + private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { - super(actorContext, encodingService, - tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(), tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer()); + super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.ruleEngineSettings = ruleEngineSettings; + this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.factory = factory; } @PostConstruct public void init() { super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); - this.factory.newInstance(); + for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + } + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue))); } @Override - protected void launchMainConsumer() { - mainConsumerExecutor.execute(() -> { + protected void stopMainConsumers() { + consumers.values().forEach(TbQueueConsumer::unsubscribe); + } + + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration) { + consumersExecutor.execute(() -> { while (!stopped) { try { - List> msgs = mainConsumer.poll(pollDuration); + List> msgs = consumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getAckStrategy()); TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { @@ -111,7 +146,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); + forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } @@ -126,7 +161,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } - mainConsumer.commit(); + consumer.commit(); } catch (Exception e) { if (!stopped) { log.warn("Failed to process messages from queue.", e); @@ -172,16 +207,27 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); - actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); + private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes = null; + if (relationTypesList != null) { + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes); + actorContext.getAppActor().tell(msg, ActorRef.noSender()); //TODO: 2.5 before release. // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } } - @Scheduled(fixedDelayString = "${queue.rule_engine.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { stats.printStats(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 781762a352..0df0517232 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -42,55 +42,48 @@ import java.util.function.Function; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractConsumerService implements ApplicationListener { +public abstract class AbstractConsumerService implements ApplicationListener { - protected volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; + protected volatile ExecutorService consumersExecutor; + protected volatile ExecutorService notificationsConsumerExecutor; protected volatile boolean stopped = false; protected final ActorSystemContext actorContext; protected final DataDecodingEncodingService encodingService; - protected final TbQueueConsumer> mainConsumer; + protected final TbQueueConsumer> nfConsumer; - public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> mainConsumer, TbQueueConsumer> nfConsumer) { + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { this.actorContext = actorContext; this.encodingService = encodingService; - this.mainConsumer = mainConsumer; this.nfConsumer = nfConsumer; } public void init(String mainConsumerThreadName, String nfConsumerThreadName) { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); } - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == getServiceType()) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); this.nfConsumer.subscribe(); launchNotificationsConsumer(); - launchMainConsumer(); + launchMainConsumers(); } protected abstract ServiceType getServiceType(); - protected abstract void launchMainConsumer(); + protected abstract void launchMainConsumers(); + + protected abstract void stopMainConsumers(); protected abstract long getNotificationPollDuration(); protected abstract long getNotificationPackProcessingTimeout(); protected void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { + notificationsConsumerExecutor.submit(() -> { while (!stopped) { try { List> msgs = nfConsumer.poll(getNotificationPollDuration()); @@ -137,16 +130,14 @@ public abstract class AbstractConsumerService ServiceType.TB_CORE.equals(key.getServiceType()))) { + if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { /* * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 0397b8cc85..77b0967966 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -113,7 +113,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio @Override @EventListener(PartitionChangeEvent.class) public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { currentPartitions.clear(); currentPartitions.addAll(partitionChangeEvent.getPartitions()); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 7a5735b843..566bf99cd3 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; @@ -42,7 +42,7 @@ import java.util.concurrent.*; @TbCoreComponent public class TbCoreTransportApiService { - private final TbCoreQueueProvider tbCoreQueueProvider; + private final TbCoreQueueFactory tbCoreQueueFactory; private final TransportApiService transportApiService; @Value("${queue.transport_api.max_pending_requests:10000}") @@ -58,16 +58,16 @@ public class TbCoreTransportApiService { private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; - public TbCoreTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { - this.tbCoreQueueProvider = tbCoreQueueProvider; + public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { + this.tbCoreQueueFactory = tbCoreQueueFactory; this.transportApiService = transportApiService; } @PostConstruct public void init() { this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); - TbQueueProducer> producer = tbCoreQueueProvider.getTransportApiResponseProducer(); - TbQueueConsumer> consumer = tbCoreQueueProvider.getTransportApiRequestConsumer(); + TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e5de845fbc..3d47db432f 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,7 +25,6 @@ - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e731bd7bac..04c9205a12 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -550,26 +550,44 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - rule_engine: + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure_percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause_between_retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: # TODO 2.5: specify correct ENV variable names. + - + name: "Main" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - + name: "HighPriority" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 215bd85930..5ecb5fb415 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -145,11 +145,10 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), - new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", null, null, null); + new TbMsgMetaData(), TbMsgDataType.JSON, "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); @@ -261,12 +260,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", null, null, null); + "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 43cdbe6ada..bc4db3b535 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -136,13 +136,12 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", - null, null, null); + "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 3c5f86a3fc..62881bd5af 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -63,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -79,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -113,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -127,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData,TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -148,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -170,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7c7816900b..7d567a31f1 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -52,13 +52,34 @@ public final class TbMsg implements Serializable { //This field is not serialized because we use queues and there is no need to do it private final TbMsgCallback callback; - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + } + + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; @@ -115,7 +136,6 @@ public final class TbMsg implements Serializable { builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); - } public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index e9b1a890c9..0e8d66aa63 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -29,7 +29,7 @@ import java.util.Set; public final class PartitionChangeMsg implements TbActorMsg { @Getter - private final ServiceKey serviceKey; + private final ServiceQueueKey serviceQueueKey; @Getter private final Set partitions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index 6f93301a48..49b30a5992 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; +import java.util.Set; /** * Created by ashvayka on 15.03.18. @@ -31,9 +32,14 @@ public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; + private final Set relationTypes; @Override public MsgType getMsgType() { return MsgType.QUEUE_TO_RULE_ENGINE_MSG; } + + public boolean isTellNext() { + return relationTypes != null && !relationTypes.isEmpty(); + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java new file mode 100644 index 0000000000..be08f054b6 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.ToString; + +import java.util.Objects; + +@ToString +public class ServiceQueue { + + public static final String MAIN = "Main"; + + private final ServiceType type; + private final String queue; + + public ServiceQueue(ServiceType type) { + this.type = type; + this.queue = MAIN; + } + + public ServiceQueue(ServiceType type, String queue) { + this.type = type; + this.queue = queue; + } + + public ServiceType getType() { + return type; + } + + public String getQueue() { + return queue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueue that = (ServiceQueue) o; + return type == that.type && + queue.equals(that.queue); + } + + @Override + public int hashCode() { + return Objects.hash(type, queue); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java index aa6eb27323..f4bfefd878 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java @@ -22,14 +22,15 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; @ToString -public class ServiceKey { +public class ServiceQueueKey { @Getter - private final ServiceType serviceType; + private final ServiceQueue serviceQueue; + @Getter private final TenantId tenantId; - public ServiceKey(ServiceType serviceType, TenantId tenantId) { - this.serviceType = serviceType; + public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { + this.serviceQueue = serviceQueue; this.tenantId = tenantId; } @@ -37,13 +38,17 @@ public class ServiceKey { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ServiceKey that = (ServiceKey) o; - return serviceType == that.serviceType && + ServiceQueueKey that = (ServiceQueueKey) o; + return serviceQueue.equals(that.serviceQueue) && Objects.equals(tenantId, that.tenantId); } @Override public int hashCode() { - return Objects.hash(serviceType, tenantId); + return Objects.hash(serviceQueue, tenantId); + } + + public ServiceType getServiceType() { + return serviceQueue.getType(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index a24a8c4b7c..0ca776126d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; public static ServiceType of(String serviceType) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java index 0d22db09e3..552eca84a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.common; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import java.util.concurrent.atomic.AtomicInteger; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java index 29a17c9e99..c372f64b3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.common; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import java.util.concurrent.atomic.AtomicInteger; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index e747b2f99d..f0039d0f38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import java.util.Set; @@ -25,10 +25,10 @@ import java.util.Set; public class ClusterTopologyChangeEvent extends ApplicationEvent { @Getter - private final Set serviceKeys; + private final Set serviceQueueKeys; - public ClusterTopologyChangeEvent(Object source, Set serviceKeys) { + public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { super(source); - this.serviceKeys = serviceKeys; + this.serviceQueueKeys = serviceQueueKeys; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index fdf79d4e1b..1337a1cc74 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -24,7 +24,8 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -53,10 +54,6 @@ public class ConsistentHashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; - @Value("${queue.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${queue.rule_engine.partitions:100}") - private Integer ruleEnginePartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @Value("${queue.partitions.virtual_nodes_size:16}") @@ -64,9 +61,9 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; - private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); @@ -85,49 +82,53 @@ public class ConsistentHashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); - partitionSizes.put(ServiceType.TB_CORE, corePartitions); - partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); - partitionTopics.put(ServiceType.TB_CORE, coreTopic); - partitionTopics.put(ServiceType.TB_RULE_ENGINE, ruleEngineTopic); + partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); + partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); } +// public Set getCurrentPartitions(ServiceType serviceType) { +// ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); +// TenantId tenantId = getSystemOrIsolatedTenantId(currentService); +// ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceType, tenantId); +// List partitions = myPartitions.get(serviceQueueKey); +// Set topicPartitions = new LinkedHashSet<>(); +// for (Integer partition : partitions) { +// TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); +// tpi.topic(partitionTopics.get(serviceType)); +// tpi.partition(partition); +// if (!tenantId.isNullUid()) { +// tpi.tenantId(tenantId); +// } +// topicPartitions.add(tpi.build()); +// } +// return topicPartitions; +// } + @Override - public Set getCurrentPartitions(ServiceType serviceType) { - ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); - TenantId tenantId = getSystemOrIsolatedTenantId(currentService); - ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); - List partitions = myPartitions.get(serviceKey); - Set topicPartitions = new LinkedHashSet<>(); - for (Integer partition : partitions) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); - tpi.partition(partition); - if (!tenantId.isNullUid()) { - tpi.tenantId(tenantId); - } - topicPartitions.add(tpi.build()); - } - return topicPartitions; + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType), tenantId, entityId); } - //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); + } + + private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - int partition = Math.abs(hash % partitionSizes.get(serviceType)); - boolean isolatedTenant = isIsolated(serviceType, tenantId); - TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceType, isolatedTenant ? tenantId : null, partition); - return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceType, tenantId, partition)); + int partition = Math.abs(hash % partitionSizes.get(serviceQueue)); + boolean isolatedTenant = isIsolated(serviceQueue, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); } @Override public void recalculatePartitions(ServiceInfo currentService, List otherServices) { logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); - - Map> circles = new HashMap<>(); + Map> circles = new HashMap<>(); addNode(circles, currentService); for (ServiceInfo other : otherServices) { TenantId tenantId = getSystemOrIsolatedTenantId(other); @@ -140,26 +141,26 @@ public class ConsistentHashPartitionService implements PartitionService { } } - ConcurrentMap> oldPartitions = myPartitions; + ConcurrentMap> oldPartitions = myPartitions; TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { - ServiceKey myServiceKey = new ServiceKey(type, myTenantId); + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myTenantId); for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceKey), i); + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); if (currentService.equals(serviceInfo)) { - ServiceKey serviceKey = new ServiceKey(type, getSystemOrIsolatedTenantId(serviceInfo)); - myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); } } }); - myPartitions.forEach((serviceKey, partitions) -> { - if (!partitions.equals(oldPartitions.get(serviceKey))) { - log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + myPartitions.forEach((serviceQueueKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) + .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) .collect(Collectors.toSet()); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); } }); tpiCache.clear(); @@ -167,14 +168,13 @@ public class ConsistentHashPartitionService implements PartitionService { if (currentOtherServices == null) { currentOtherServices = new ArrayList<>(otherServices); } else { - Set changes = new HashSet<>(); - Map> currentMap = getServiceKeyListMap(currentOtherServices); - Map> newMap = getServiceKeyListMap(otherServices); + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); currentOtherServices = otherServices; currentMap.forEach((key, list) -> { if (!list.equals(newMap.get(key))) { changes.add(key); - } }); currentMap.keySet().forEach(newMap::remove); @@ -214,13 +214,20 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } } }); return currentMap; @@ -230,20 +237,20 @@ public class ConsistentHashPartitionService implements PartitionService { return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { - return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { + return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); + tpi.topic(partitionTopics.get(serviceQueue)); tpi.partition(partition); - ServiceKey myPartitionsSearchKey; - if (isIsolated(serviceType, tenantId)) { + ServiceQueueKey myPartitionsSearchKey; + if (isIsolated(serviceQueue, tenantId)) { tpi.tenantId(tenantId); - myPartitionsSearchKey = new ServiceKey(serviceType, tenantId); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); } else { - myPartitionsSearchKey = new ServiceKey(serviceType, new TenantId(TenantId.NULL_UUID)); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); } List partitions = myPartitions.get(myPartitionsSearchKey); if (partitions != null) { @@ -254,8 +261,8 @@ public class ConsistentHashPartitionService implements PartitionService { return tpi.build(); } - private boolean isIsolated(ServiceType serviceType, TenantId tenantId) { - return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { + return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceQueue.getType()); } private void logServiceInfo(TransportProtos.ServiceInfo server) { @@ -271,13 +278,24 @@ public class ConsistentHashPartitionService implements PartitionService { return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); } - private void addNode(Map> circles, ServiceInfo instance) { + private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 2ced989525..15cf33bb82 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -17,12 +17,16 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PostConstruct; import java.net.InetAddress; @@ -49,6 +53,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.tenant_id:}") private String tenantIdStr; + @Autowired(required = false) + private TbQueueRuleEngineSettings ruleEngineSettings; + private List serviceTypes; private ServiceInfo serviceInfo; @@ -78,13 +85,18 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } builder.setTenantIdMSB(tenantId.getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); - serviceInfo = builder.build(); - } + if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { + for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { + TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() + .setName(queue.getName()) + .setTopic(queue.getTopic()) + .setPartitions(queue.getPartitions()).build(); + builder.addRuleEngineQueues(queueInfo); + } + } - @Override - public List getSupportedServiceTypes() { - return serviceTypes; + serviceInfo = builder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 05218c68d4..19b8f665a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -17,7 +17,8 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; @@ -26,13 +27,17 @@ import java.util.Set; public class PartitionChangeEvent extends ApplicationEvent { @Getter - private final ServiceKey serviceKey; + private final ServiceQueueKey serviceQueueKey; @Getter private final Set partitions; - public PartitionChangeEvent(Object source, ServiceKey serviceKey, Set partitions) { + public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { super(source); - this.serviceKey = serviceKey; + this.serviceQueueKey = serviceQueueKey; this.partitions = partitions; } + + public ServiceType getServiceType() { + return serviceQueueKey.getServiceQueue().getType(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b22b53f93e..e3b3e76b80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -29,10 +29,10 @@ import java.util.Set; */ public interface PartitionService { - Set getCurrentPartitions(ServiceType serviceType); - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 8e20b35bde..053dd644c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,15 +15,10 @@ */ package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import java.util.List; - public interface TbServiceInfoProvider { - List getSupportedServiceTypes(); - String getServiceId(); ServiceInfo getServiceInfo(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java index 2d51bc581a..37661e85e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -17,13 +17,13 @@ package org.thingsboard.server.queue.discovery; import lombok.AllArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.ServiceQueue; import java.util.Objects; @AllArgsConstructor public class TopicPartitionInfoKey { - private ServiceType serviceType; + private ServiceQueue serviceQueue; private TenantId isolatedTenantId; private int partition; @@ -33,12 +33,12 @@ public class TopicPartitionInfoKey { if (o == null || getClass() != o.getClass()) return false; TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; return partition == that.partition && - serviceType == that.serviceType && + serviceQueue.equals(that.serviceQueue) && Objects.equals(isolatedTenantId, that.isolatedTenantId); } @Override public int hashCode() { - return Objects.hash(serviceType, isolatedTenantId, partition); + return Objects.hash(serviceQueue, isolatedTenantId, partition); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 206a71d85e..72d5731139 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -21,14 +21,15 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -36,7 +37,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; @@ -47,12 +48,12 @@ public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; - public AwsSqsMonolithQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings) { + public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,64 +65,64 @@ public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 052d3fc0e3..1fba780395 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -26,10 +26,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -40,7 +40,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") -public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { +public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbAwsSqsSettings sqsSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -52,12 +52,12 @@ public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueAdmin admin; - public AwsSqsTbCoreQueueProvider(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -68,50 +68,50 @@ public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index d9f2cec9f4..4181a9442d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -24,12 +24,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -37,7 +38,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") -public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; @@ -46,10 +47,10 @@ public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; - public AwsSqsTbRuleEngineQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings) { + public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -59,37 +60,37 @@ public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 01360ebed8..09ca194724 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -23,8 +23,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -36,17 +36,17 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { +public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; - public AwsSqsTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider) { + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; @@ -55,7 +55,7 @@ public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbAwsSqsProducerTemplate> producerTemplate = new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); @@ -76,17 +76,17 @@ public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 5e1bab5e69..b40876d661 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -26,29 +26,30 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryMonolithQueueFactory(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -56,57 +57,57 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java similarity index 77% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 17e38b244e..9914cf65f9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -20,12 +20,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -39,17 +39,17 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvider { +public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTbTransportQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryTbTransportQueueFactory(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -57,7 +57,7 @@ public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvide } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); @@ -73,17 +73,17 @@ public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvide } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java similarity index 82% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 56cf4fb55f..8b84d08bff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,21 +26,22 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -50,12 +51,12 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaMonolithQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -66,7 +67,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -75,7 +76,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -84,7 +85,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -93,7 +94,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); @@ -102,7 +103,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -111,18 +112,19 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -133,7 +135,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); @@ -144,7 +146,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -155,7 +157,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); @@ -166,7 +168,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 110a98b3b9..7083553abb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,10 +26,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") -public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { +public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -48,11 +48,11 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings) { + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -62,7 +62,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -71,7 +71,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -80,7 +80,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -89,7 +89,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); @@ -98,7 +98,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -107,7 +107,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); @@ -118,7 +118,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -129,7 +129,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); @@ -140,7 +140,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java similarity index 80% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 7f939ee11d..9b8a9c4ba0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -24,19 +24,20 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") -public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -44,10 +45,10 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings) { + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -56,7 +57,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -65,7 +66,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -74,7 +75,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -84,7 +85,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); @@ -93,7 +94,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -102,18 +103,19 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 2213889964..12f677af9d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -19,12 +19,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { +public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -50,12 +50,12 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -65,7 +65,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); @@ -90,7 +90,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); @@ -99,7 +99,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); @@ -108,7 +108,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 2dc9549679..4aa35e5d68 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,11 +27,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,10 +39,11 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -55,13 +56,13 @@ public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE private TbQueueProducer> tbCoreProducer; - public PubSubMonolithQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -73,65 +74,65 @@ public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java similarity index 78% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 4c89b35dec..da2fe3acdf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -27,9 +27,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") -public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { +public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -48,12 +48,12 @@ public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - public PubSubTbCoreQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueAdmin admin, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -63,51 +63,51 @@ public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 3f707235fa..da65d583c1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -25,19 +25,20 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") -public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -46,12 +47,12 @@ public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - public PubSubTbRuleEngineQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueAdmin admin, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -61,39 +62,39 @@ public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index b1a7efc950..f15faa11da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -25,12 +25,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class PubSubTransportQueueProvider implements TbTransportQueueProvider { +public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -52,12 +52,12 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueAdmin admin; - public PubSubTransportQueueProvider(TbPubSubSettings pubSubSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -68,7 +68,7 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), @@ -86,17 +86,17 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index e74c07152b..1a3ba1ab98 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -29,70 +29,70 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueProvider { +public interface TbCoreQueueFactory { /** * Used to push messages to instances of TB Transport Service * * @return */ - TbQueueProducer> getTransportNotificationsMsgProducer(); + TbQueueProducer> createTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); /** * Used to push notifications to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); /** * Used to push notifications to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> createTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service * * @return */ - TbQueueConsumer> getToCoreMsgConsumer(); + TbQueueConsumer> createToCoreMsgConsumer(); /** * Used to consume high priority messages by TB Core Service * * @return */ - TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + TbQueueConsumer> createToCoreNotificationsMsgConsumer(); /** * Used to consume Transport API Calls * * @return */ - TbQueueConsumer> getTransportApiRequestConsumer(); + TbQueueConsumer> createTransportApiRequestConsumer(); /** * Used to push replies to Transport API Calls * * @return */ - TbQueueProducer> getTransportApiResponseProducer(); + TbQueueProducer> createTransportApiResponseProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index c83159242d..9fe5de0bcc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -27,24 +27,24 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { - private final TbCoreQueueProvider tbQueueProvider; + private final TbCoreQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; - public TbCoreQueueProducerProvider(TbCoreQueueProvider tbQueueProvider) { + public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index ed7dd43f2a..69de7ca3e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -27,7 +27,7 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { - private final TbRuleEngineQueueProvider tbQueueProvider; + private final TbRuleEngineQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; @@ -35,17 +35,17 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; - public TbRuleEngineProducerProvider(TbRuleEngineQueueProvider tbQueueProvider) { + public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index fafa7da9e3..300fb986a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -23,60 +23,63 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueProvider { +public interface TbRuleEngineQueueFactory { /** * Used to push messages to instances of TB Transport Service * * @return */ - TbQueueProducer> getTransportNotificationsMsgProducer(); + TbQueueProducer> createTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); /** * Used to push notifications to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); /** * Used to push notifications to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> createTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service * * @return + * @param configuration */ - TbQueueConsumer> getToRuleEngineMsgConsumer(); + //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary + TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); /** * Used to consume high priority messages by TB Core Service * * @return */ - TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer(); + TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java similarity index 78% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index 09521692f6..dc1d2c449c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -25,14 +25,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -public interface TbTransportQueueProvider { +public interface TbTransportQueueFactory { - TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); + TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); - TbQueueConsumer> getTransportNotificationsConsumer(); + TbQueueConsumer> createTransportNotificationsConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index d78f10ef74..0ebd59ec24 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -28,19 +28,19 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { - private final TbTransportQueueProvider tbQueueProvider; + private final TbTransportQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; - public TbTransportQueueProducerProvider(TbTransportQueueProvider tbQueueProvider) { + public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 4957b5ed5b..1c260c754d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..48eeb5ba2a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Data +@Configuration +@ConfigurationProperties(prefix = "queue.rule-engine") +public class TbQueueRuleEngineSettings { + + private String topic; + private List queues; + + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. See how ther are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + @PostConstruct + public void validate() { + queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { + log.warn("Main queue is not configured in thingsboard.yml"); + return new RuntimeException("No \"Main\" queue configured!"); + }); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 263cf262a0..5ee58ab17e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index 0f2ac55862..50da4f4a1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java new file mode 100644 index 0000000000..c1a8fd883d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; + +@Data +public class TbRuleEngineQueueAckStrategyConfiguration { + +// @Value("${type}") + private String type; +// @Value("${retries:3}") + private int retries; +// @Value("${failure_percentage:0}") + private double failurePercentage; +// @Value("${pause_between_retries:3}") + private long pauseBetweenRetries; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java similarity index 71% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index fc3a36a88a..f89a615d7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -13,19 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; @Data -@Component -public class TbQueueRuleEngineSettings { +public class TbRuleEngineQueueConfiguration { - @Value("${queue.rule_engine.topic}") + private String name; private String topic; - - @Value("${queue.rule_engine.partitions}") + private int pollInterval; private int partitions; + private String packProcessingTimeout; + private TbRuleEngineQueueAckStrategyConfiguration ackStrategy; + } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 87c256174c..55d376b63b 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -19,6 +19,12 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +message QueueInfo { + string name = 1; + string topic = 2; + int32 partitions = 3; +} + /** * Service Discovery Data Structures; */ @@ -27,6 +33,7 @@ message ServiceInfo { repeated string serviceTypes = 2; int64 tenantIdMSB = 3; int64 tenantIdLSB = 4; + repeated QueueInfo ruleEngineQueues = 5; } /** @@ -380,6 +387,7 @@ message ToRuleEngineMsg { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; bytes tbMsg = 3; + repeated string relationTypes = 4; } message ToRuleEngineNotificationMsg { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3a707d11a7..d107bbbeeb 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -23,9 +23,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.queue.TbQueueCallback; @@ -46,7 +44,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -92,7 +90,7 @@ public class DefaultTransportService implements TransportService { private int notificationsPollDuration; private final Gson gson = new Gson(); - private final TbTransportQueueProvider queueProvider; + private final TbTransportQueueFactory queueProvider; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; @@ -112,7 +110,7 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueProvider queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.queueProvider = queueProvider; this.producerProvider = producerProvider; this.partitionService = partitionService; @@ -128,10 +126,10 @@ public class DefaultTransportService implements TransportService { this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { @@ -251,8 +249,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("deviceType", sessionInfo.getDeviceType()); metaData.putValue("ts", tsKv.getTs() + ""); JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), - deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); sendToRuleEngine(tenantId, tbMsg, packCallback); } } @@ -268,8 +265,7 @@ public class DefaultTransportService implements TransportService { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("deviceName", sessionInfo.getDeviceName()); metaData.putValue("deviceType", sessionInfo.getDeviceType()); - TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, - TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 38914f4553..f46878ca68 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -45,27 +46,80 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { + /* + * + * METHODS TO CONTROL THE MESSAGE FLOW + * + */ + + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using specified relationType. + * + * @param msg + * @param relationType + */ void tellNext(TbMsg msg, String relationType); - void tellNext(TbMsg msg, String relationType, Throwable th); - + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using one of specified relationTypes. + * + * @param msg + * @param relationTypes + */ void tellNext(TbMsg msg, Set relationTypes); + /** + * Sends message to the current Rule Node with specified delay in milliseconds. + * Note: this message is not queued and may be lost in case of a server restart. + * + * @param msg + */ void tellSelf(TbMsg msg, long delayMs); - boolean isLocalEntity(EntityId entityId); - + /** + * Notifies Rule Engine about failure to process current message. + * + * @param msg - message + * @param th - exception + */ void tellFailure(TbMsg msg, Throwable th); - void updateSelf(RuleNode self); + /** + * Puts new message to queue for processing by the Root Rule Chain + * + * @param msg - message + */ + void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); + + /** + * Puts new message to custom queue for processing + * + * @param msg - message + */ + void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String relationType); + + void enqueueForTellNext(TbMsg msg, Set relationTypes); + + void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - void sendTbMsgToRuleEngine(TbMsg msg); + void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void ack(TbMsg tbMsg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -77,8 +131,17 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); + // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); + /* + * + * METHODS TO PROCESS THE MESSAGES + * + */ + + boolean isLocalEntity(EntityId entityId); + RuleNodeId getSelfId(); TenantId getTenantId(); @@ -129,7 +192,7 @@ public interface TbContext { void logJsEvalFailure(); - String getNodeId(); + String getServiceId(); RuleChainTransactionService getRuleChainTransactionService(); @@ -139,7 +202,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); + //TODO 2.5: - need to remove this. RedisTemplate getRedisTemplate(); - String getServerAddress(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index 6320d5625f..afc4181f70 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import javax.script.ScriptException; - import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -63,16 +61,16 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), + throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); } }, - t -> ctx.tellFailure(msg, t) - , ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 3ee2127fde..7b84fa06a9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -122,7 +122,9 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index bcfabf1100..96fdd2a11e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -186,7 +186,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), + throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -201,7 +203,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), + throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -215,7 +219,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 1952985e53..282ca00639 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,7 +68,7 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); @@ -76,11 +76,12 @@ public class TbMsgCountNode implements TbNode { metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); //TODO 2.5: Callback? - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, null); - ctx.tellNext(tbMsg, SUCCESS); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); + ctx.enqueueForTellNext(tbMsg, SUCCESS, null, null); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 0ef9c4d46a..5da20f5c35 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -73,7 +73,7 @@ public class TbMsgDelayNode implements TbNode { TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); } else { - ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index fe765abdf9..e5b3f4a1c7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -66,7 +66,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -108,7 +108,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 216fea54e3..7c3705fa1f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.Properties; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; @Slf4j @RuleNode( @@ -55,7 +54,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java index 3349706221..015cced6c6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -105,7 +105,7 @@ class TbRedisQueueProcessor { } private String constructRedisKey() { - return ctx.getServerAddress() + ctx.getSelfId(); + return ctx.getServiceId() + ctx.getSelfId(); } private int validateMaxQueueSize() { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index a6bd7b6164..b8eff0901a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -58,8 +58,7 @@ public class TbSynchronizationBeginNode implements TbNode { TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); //TODO 2.5: Callback? - TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), null); + TbMsg tbMsg = TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index e0b76f1cda..c2177a9c9a 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -101,7 +101,7 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -143,7 +143,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +166,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -211,7 +211,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -258,7 +258,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index c840e708eb..ac91e0548c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -59,7 +59,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -72,7 +72,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -85,7 +85,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 4934e14062..f96797b69e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -66,7 +66,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index 8315d8e40b..c07bee7d79 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -63,7 +63,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0169cadcb2..1575823988 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -103,7 +103,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -128,7 +128,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -153,7 +153,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -167,7 +167,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); entityAttributeFetched(customerId); } @@ -178,7 +178,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -193,7 +193,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -208,7 +208,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -235,7 +235,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 1ed66fc786..fa845c4a18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -92,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -120,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -147,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 9a315f893b..314ee78c9c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -63,8 +63,8 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId, null); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); + TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); @@ -85,7 +85,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); From c52d0d26d305a6d90f4e78c76fdb7dde5cfa1a35 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 19:23:33 +0300 Subject: [PATCH 030/102] Refactoring of Message Routing --- .../actors/ruleChain/DefaultTbContext.java | 5 ++++ .../rule/engine/api/TbContext.java | 9 ++++++ .../engine/action/TbAbstractAlarmNode.java | 2 ++ .../action/TbAbstractCustomerActionNode.java | 2 +- .../TbCopyAttributesToEntityViewNode.java | 8 +++--- .../rule/engine/action/TbLogNode.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 2 +- .../TbSaveToCustomCassandraTableNode.java | 6 ++-- .../rule/engine/aws/sns/TbSnsNode.java | 7 ++--- .../rule/engine/aws/sqs/TbSqsNode.java | 9 ++---- .../rule/engine/debug/TbMsgGeneratorNode.java | 2 +- .../rule/engine/delay/TbMsgDelayNode.java | 5 ++-- .../engine/filter/TbCheckAlarmStatusNode.java | 3 +- .../engine/filter/TbMsgTypeFilterNode.java | 2 +- .../engine/filter/TbMsgTypeSwitchNode.java | 2 +- .../filter/TbOriginatorTypeFilterNode.java | 2 +- .../rule/engine/gcp/pubsub/TbPubSubNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 26 +++++++++++------ .../rule/engine/kafka/TbKafkaNode.java | 5 ++-- .../rule/engine/mail/TbSendEmailNode.java | 3 +- .../metadata/TbAbstractGetAttributesNode.java | 3 +- .../TbAbstractGetEntityDetailsNode.java | 2 +- .../engine/metadata/TbEntityGetAttrNode.java | 5 ++-- .../metadata/TbGetOriginatorFieldsNode.java | 4 +-- .../engine/metadata/TbGetTelemetryNode.java | 3 +- .../rule/engine/mqtt/TbMqttNode.java | 4 +-- .../rule/engine/rabbitmq/TbRabbitMqNode.java | 4 +-- .../rule/engine/rest/TbHttpClient.java | 2 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 1 + .../rule/engine/rpc/TbSendRPCRequestNode.java | 8 ++++-- .../engine/telemetry/TbMsgAttributesNode.java | 1 - .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +-- .../telemetry/TelemetryNodeCallback.java | 4 +-- .../transform/TbAbstractTransformNode.java | 2 +- .../rule/engine/action/TbAlarmNodeTest.java | 28 ++++++++++++++----- .../engine/filter/TbJsFilterNodeTest.java | 11 ++++++-- .../engine/filter/TbJsSwitchNodeTest.java | 9 ++++-- .../engine/mail/TbMsgToEmailNodeTest.java | 1 - .../TbGetCustomerAttributeNodeTest.java | 13 +++++++-- .../transform/TbTransformMsgNodeTest.java | 10 +++++-- 40 files changed, 137 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 2f31d1dc35..f8ac564e42 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -90,6 +90,11 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } + @Override + public void tellSuccess(TbMsg msg) { + tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); + } + @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index f46878ca68..72a552cb4f 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -59,6 +59,15 @@ public interface TbContext { * */ + /** + * Indicates that message was successfully processed by the rule node. + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using "Success" relationType. + * + * @param msg + */ + void tellSuccess(TbMsg msg); + /** * Sends message to all Rule Nodes in the Rule Chain * that are connected to the current Rule Node using specified relationType. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index afc4181f70..b3344105bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -68,6 +68,8 @@ public abstract class TbAbstractAlarmNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 7b84fa06a9..bdf8c5fe1e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellNext(msg, "Success"), + m -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index f009b737e9..c5dc5cc5f4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,13 +117,14 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (filteredAttributes != null && !filteredAttributes.isEmpty()) { + if (!filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } + ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -135,8 +136,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); - ctx.tellNext(updMsg, SUCCESS); + ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 62410e3052..2831e890d6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 282ca00639..10e664bc16 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -77,7 +77,7 @@ public class TbMsgCountNode implements TbNode { //TODO 2.5: Callback? TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); - ctx.enqueueForTellNext(tbMsg, SUCCESS, null, null); + ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 50471b366a..aa7bfa81d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -105,10 +105,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - withCallback(save(msg, ctx), aVoid -> { - ctx.tellNext(msg, SUCCESS); - }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) { + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index 59d27018d7..d43ebb26ef 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,11 +74,8 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 37f4e4baee..3cc6074165 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,13 +80,10 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 93114c28e0..4f469678a2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.tellNext(m, SUCCESS); + ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 5da20f5c35..1807c5c4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,13 +65,14 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.tellNext(pendingMsg, SUCCESS); + ctx.enqueueForTellNext(pendingMsg, SUCCESS); } } else { - if(pendingMsgs.size() < config.getMaxPendingMsgs()) { + if (pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); + ctx.ack(msg); } else { ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index e0ec9300d4..8935cfc1f5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -76,14 +76,13 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } - if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index dd55ef2584..3799e60f73 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index d0fe593711..07a0508a87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index ac3db9968f..de3a48142e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 8e3bcdb7dc..5a8b92cf80 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index e5b3f4a1c7..832a86b5cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,11 +47,12 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig") + configDirective = "tbActionNodeGpsGeofencingConfig" +) public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -78,17 +79,26 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } else { + if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } } } + if (!told) { + ctx.tellSuccess(msg); + } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 7c3705fa1f..2c487f67b9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -64,8 +64,7 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties() - .forEach((k,v) -> properties.put(k, v)); + config.getOtherProperties().forEach(properties::put); } try { this.producer = new KafkaProducer<>(properties); @@ -75,7 +74,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { producer.send(new ProducerRecord<>(topic, msg.getData()), diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index 1c41c2a124..ee46b81116 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -79,7 +78,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellNext(msg, SUCCESS), + ok -> ctx.tellSuccess(msg), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index be4e6b4e30..8aadbd6b32 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -42,7 +42,6 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -100,7 +99,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index b6fdaa565a..4ca0bf2dff 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -57,7 +57,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), + ctx::tellSuccess, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index ca2c99fe7a..8f19cba741 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -50,8 +50,7 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback( - findEntityAsync(ctx, msg.getOriginator()), + withCallback(findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -88,7 +87,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 759007789a..160ea9adb1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -49,10 +49,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index caef71a4db..a3d84b25db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,8 +106,7 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - ctx.tellNext(newMsg, SUCCESS); + ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 755cd27a33..b13f72b238 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellNext(msg, TbRelationTypes.SUCCESS); + ctx.tellSuccess(msg); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 3c3bde2915..4a61cf63f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + ctx::tellSuccess, t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 65eea0b4c9..5b30f4792c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -113,7 +113,7 @@ class TbHttpClient { queueProcessor.resetCounter(); } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } else { if (config.isUseRedisQueueForMsgPersistence()) { processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 0c5c7d188e..393ef0bd7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -59,6 +59,7 @@ public class TbSendRPCReplyNode implements TbNode { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index e4eeef2e5f..88b060fe5c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -112,13 +112,15 @@ public class TbSendRPCRequestNode implements TbNode { ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.enqueueForTellNext(next, TbRelationTypes.FAILURE); } }); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 4ddd082f3e..8d80e17f87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,7 +61,6 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } - String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index ce7edf1c5f..14565ac475 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap == null) { + if (tsKvMap.isEmpty()) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index d545d41d7b..bd15c5bd83 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; - /** * Created by ashvayka on 02.04.18. */ @@ -34,7 +32,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index e768f699cd..84a6b76920 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellNext(m, SUCCESS); + ctx.tellSuccess(m); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index c2177a9c9a..2fcdc6c838 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -26,18 +26,19 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -47,13 +48,26 @@ import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index ac91e0548c..b0f64fb980 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,7 +19,6 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -28,7 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -39,7 +41,10 @@ import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index f96797b69e..9286a06da6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,7 +28,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -41,7 +44,9 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index c07bee7d79..70f58c5989 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -37,7 +37,6 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 1575823988..211459b84c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,8 +31,17 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 314ee78c9c..3d40f3acd0 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,7 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -39,7 +42,10 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) From ff3fd89acedf0b356db9638adbf38c3c8e41abfe Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 19:40:38 +0300 Subject: [PATCH 031/102] Added ability to process tellNext messages from Queue --- .../actors/ruleChain/DefaultTbContext.java | 6 +- .../RuleChainActorMessageProcessor.java | 66 ++++++++++--------- .../thingsboard/server/common/msg/TbMsg.java | 6 +- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index f8ac564e42..7178b94dfa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; @@ -175,6 +176,9 @@ class DefaultTbContext implements TbContext { } private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); + RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); + tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 00d9d85b1c..3ab67bf81f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,18 +34,18 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import java.util.ArrayList; import java.util.Collections; @@ -194,25 +194,29 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes) { try { checkActive(); EntityId entityId = msg.getOriginator(); TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - RuleNodeId originatorNodeId = envelope.getOriginator(); List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .filter(r -> contains(relationTypes, r.getType())) .collect(Collectors.toList()); int relationsCount = relations.size(); if (relationsCount == 0) { log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (envelope.getRelationTypes().contains(TbRelationTypes.FAILURE)) { - log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, envelope.getOriginator().getId()); + if (relationTypes.contains(TbRelationTypes.FAILURE)) { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); //TODO 2.5: Introduce our own RuleEngineFailureException to track what is wrong - msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + envelope.getOriginator().getId().toString() + "]")); + msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); } else { msg.getCallback().onSuccess(); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7d567a31f1..7230804773 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -73,6 +73,10 @@ public final class TbMsg implements Serializable { data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); } + public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); From 45bd764e6fec0f994f5345a20653c185f5b190ad Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 7 Apr 2020 11:45:28 +0300 Subject: [PATCH 032/102] Azure ServiceBus queue * created event hubs queue * created servicebus queue * refactored * refactored --- .../actors/ruleChain/DefaultTbContext.java | 2 +- .../RuleChainActorMessageProcessor.java | 2 +- .../src/main/resources/thingsboard.yml | 7 +- .../thingsboard/server/common/msg/TbMsg.java | 2 +- common/queue/pom.xml | 4 + .../azure/servicebus/TbServiceBusAdmin.java | 79 +++++++ .../TbServiceBusConsumerTemplate.java | 210 ++++++++++++++++++ .../TbServiceBusProducerTemplate.java | 110 +++++++++ .../servicebus/TbServiceBusSettings.java | 37 +++ .../queue/common/DefaultTbQueueMsg.java | 13 +- .../ConsistentHashPartitionService.java | 1 - .../provider/AwsSqsMonolithQueueFactory.java | 8 +- .../provider/AwsSqsTbCoreQueueFactory.java | 22 +- .../AwsSqsTbRuleEngineQueueFactory.java | 7 +- .../provider/AwsSqsTransportQueueFactory.java | 28 ++- .../InMemoryMonolithQueueFactory.java | 8 +- .../InMemoryTbTransportQueueFactory.java | 18 +- .../provider/KafkaMonolithQueueFactory.java | 8 +- .../provider/KafkaTbCoreQueueFactory.java | 6 +- .../KafkaTbRuleEngineQueueFactory.java | 4 +- .../provider/PubSubMonolithQueueFactory.java | 10 +- .../ServiceBusMonolithQueueFactory.java | 134 +++++++++++ .../ServiceBusTbCoreQueueProvider.java | 116 ++++++++++ .../ServiceBusTbRuleEngineQueueFactory.java | 99 +++++++++ .../ServiceBusTransportQueueFactory.java | 94 ++++++++ .../pubsub/TbPubSubConsumerTemplate.java | 11 +- .../pubsub/TbPubSubProducerTemplate.java | 3 +- .../queue/sqs/TbAwsSqsConsumerTemplate.java | 11 +- .../queue/sqs/TbAwsSqsProducerTemplate.java | 9 - pom.xml | 6 + .../src/main/resources/tb-mqtt-transport.yml | 7 +- 31 files changed, 978 insertions(+), 98 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7178b94dfa..81444769d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 3ab67bf81f..3a03cdadde 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 04c9205a12..cb562a7f52 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -517,7 +517,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -537,6 +537,11 @@ queue: ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7230804773..10c372d546 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 0a61eb0cf6..f55f69ece1 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -60,6 +60,10 @@ com.google.cloud google-cloud-pubsub + + com.microsoft.azure + azure-servicebus + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java new file mode 100644 index 0000000000..fb9cd89a8d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusAdmin implements TbQueueAdmin { + + private final Set queues = ConcurrentHashMap.newKeySet(); + + private final ManagementClient client; + + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + ConnectionStringBuilder builder = new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + "queues", + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + + client = new ManagementClient(builder); + + try { + client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to get queues.", e); + throw new RuntimeException("Failed to get queues.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + if (queues.contains(topic)) { + return; + } + + try { + client.createQueue(topic); + queues.add(topic); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to create queue: [{}]", topic, e); + } + } + + @PreDestroy + private void destroy() { + try { + client.close(); + } catch (IOException e) { + log.error("Failed to close ManagementClient."); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java new file mode 100644 index 0000000000..cca599d59a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.microsoft.azure.servicebus.TransactionContext; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; +import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; +import com.microsoft.azure.servicebus.primitives.MessagingEntityType; +import com.microsoft.azure.servicebus.primitives.MessagingFactory; +import com.microsoft.azure.servicebus.primitives.SettleModePair; +import lombok.extern.slf4j.Slf4j; +import org.apache.qpid.proton.amqp.messaging.Data; +import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; +import org.apache.qpid.proton.amqp.transport.SenderSettleMode; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbServiceBusConsumerTemplate implements TbQueueConsumer { + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbServiceBusSettings serviceBusSettings; + + private final Gson gson = new Gson(); + + private Set receivers; + private volatile Set partitions; + private volatile boolean subscribed; + private volatile boolean stopped = false; + private Map> pendingMessages = new ConcurrentHashMap<>(); + private volatile int messagesPerQueue; + + public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.serviceBusSettings = serviceBusSettings; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + receivers.forEach(CoreMessageReceiver::closeAsync); + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); + subscribed = true; + } + + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to parse message.", e); + throw new RuntimeException("Failed to parse message.", e); + } + }).collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + private void createReceivers() { + List> receiverFutures = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .map(queue -> { + MessagingFactory factory; + try { + factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to create factory for the queue [{}]", queue); + throw new RuntimeException("Failed to create the factory", e); + } + + return CoreMessageReceiver.create(factory, queue, queue, 0, + new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), + MessagingEntityType.QUEUE); + }).collect(Collectors.toList()); + + try { + receivers = new HashSet<>(fromList(receiverFutures).get()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to create receivers", e); + } + } + } + + private ConnectionStringBuilder createConnection(String queue) { + admin.createTopicIfNotExists(queue); + return new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + queue, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + } + + private CompletableFuture> fromList(List> futures) { + CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; + futures.toArray(arrayFuture); + + return CompletableFuture + .allOf(arrayFuture) + .thenApply(v -> futures + .stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + } + + @Override + public void commit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); + } + + private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java new file mode 100644 index 0000000000..5d9a931378 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java @@ -0,0 +1,110 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.Message; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.ReceiveMode; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +public class TbServiceBusProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbServiceBusSettings serviceBusSettings; + private final Map clients = new HashMap<>(); + private ExecutorService executorService; + + public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.serviceBusSettings = serviceBusSettings; + executorService = Executors.newSingleThreadExecutor(); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); + CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); + future.whenCompleteAsync((success, err) -> { + if (err != null) { + callback.onFailure(err); + } else { + callback.onSuccess(null); + } + }, executorService); + } + + @Override + public void stop() { + clients.forEach((t, client) -> { + try { + client.close(); + } catch (ServiceBusException e) { + log.error("Failed to close QueueClient.", e); + } + }); + + if (executorService != null) { + executorService.shutdownNow(); + } + } + + private QueueClient getClient(String topic) { + return clients.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + ConnectionStringBuilder builder = + new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + topic, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + try { + return new QueueClient(builder, ReceiveMode.PEEKLOCK); + } catch (InterruptedException | ServiceBusException e) { + log.error("Failed to create new client for the Queue: [{}]", topic, e); + throw new RuntimeException("Failed to create new client for the Queue", e); + } + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java new file mode 100644 index 0000000000..d872dcec0b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +@Component +@Data +public class TbServiceBusSettings { + @Value("${queue.service_bus.namespace_name}") + private String namespaceName; + @Value("${queue.service_bus.sas_key_name}") + private String sasKeyName; + @Value("${queue.service_bus.sas_key}") + private String sasKey; + @Value("${queue.service_bus.max_messages}") + private int maxMessages; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index 0e816ae59b..c7e439ef7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.common; -import com.google.gson.annotations.Expose; import lombok.Data; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgHeaders; @@ -26,12 +25,20 @@ import java.util.UUID; public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; + private DefaultTbQueueMsgHeaders headers; + public DefaultTbQueueMsg(UUID key, byte[] data) { this.key = key; this.data = data; } - @Expose(serialize = false, deserialize = false) - private TbQueueMsgHeaders headers; + public DefaultTbQueueMsg(TbQueueMsg msg) { + this.key = msg.getKey(); + this.data = msg.getData(); + DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + msg.getHeaders().getData().forEach(headers::put); + this.headers = headers; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 1337a1cc74..f36ae70dfa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -36,7 +36,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 72d5731139..9f70345301 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -21,14 +21,14 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 1fba780395..770e7fa65c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -18,21 +18,22 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -48,8 +49,6 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin admin; public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, @@ -78,7 +77,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @@ -88,7 +87,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @@ -99,15 +98,16 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 4181a9442d..f87f108453 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -24,12 +24,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; @@ -86,7 +86,8 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 09ca194724..4196b226ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -18,16 +18,20 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -55,17 +59,17 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbAwsSqsProducerTemplate> producerTemplate = + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); - TbAwsSqsConsumerTemplate> consumerTemplate = + TbAwsSqsConsumerTemplate> consumerTemplate = new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); templateBuilder.queueAdmin(admin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); @@ -76,18 +80,18 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index b40876d661..a23c93e015 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -26,14 +26,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Slf4j diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 9914cf65f9..c779127e1f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -19,22 +19,22 @@ import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 8b84d08bff..fa36f8284b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,17 +26,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 7083553abb..5e46d67eed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,16 +26,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 9b8a9c4ba0..eef10f35dc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -24,15 +24,15 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 4aa35e5d68..f00bd7caef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,11 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,6 +35,10 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @@ -54,8 +54,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private TbQueueProducer> tbCoreProducer; - public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java new file mode 100644 index 0000000000..914e200fc9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java new file mode 100644 index 0000000000..334af58133 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") +public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public ServiceBusTbCoreQueueProvider(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..3ceb837606 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") +public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java new file mode 100644 index 0000000000..d38a4389a3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = + new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + + TbQueueConsumer> consumerTemplate = + new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 4109e72070..5dc795739a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -20,7 +20,6 @@ import com.google.api.core.ApiFutures; import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import com.google.pubsub.v1.AcknowledgeRequest; @@ -36,15 +35,12 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -139,7 +135,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); subscriptionNames.forEach(admin::createTopicIfNotExists); consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages()/subscriptionNames.size(); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); subscribed = true; } List messages; @@ -217,11 +213,6 @@ public class TbPubSubConsumerTemplate implements TbQueueCo public T decode(PubsubMessage message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - Map headerMap = gson.fromJson(message.getAttributesMap().get("headers"), new TypeToken>() { - }.getType()); - headerMap.forEach(headers::put); - msg.setHeaders(headers); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index fb9bc8b189..2cd2e1054e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -71,7 +71,6 @@ public class TbPubSubProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); pubsubMessageBuilder.setData(getMsg(msg)); - pubsubMessageBuilder.putAttributes("headers", gson.toJson(msg.getHeaders().getData())); Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); @@ -110,7 +109,7 @@ public class TbPubSubProducerTemplate implements TbQueuePr } private ByteString getMsg(T msg) { - String json = gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData())); + String json = gson.toJson(new DefaultTbQueueMsg(msg)); return ByteString.copyFrom(json.getBytes()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index b4a2f84b83..3e71388844 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -23,7 +23,6 @@ import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; -import com.google.common.reflect.TypeToken; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,15 +37,12 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -161,7 +157,7 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo if (stopped) { log.info("[{}] Aws SQS consumer is stopped.", topic); } else { - log.error("Failed to pool messages.", e); + log.error("Failed to pool messages.", e); } } } @@ -214,11 +210,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo public T decode(Message message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { - }.getType()); - headerMap.forEach(headers::put); - msg.setHeaders(headers); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index a5707c845d..d304953ef4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -20,7 +20,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.MessageAttributeValue; import com.amazonaws.services.sqs.model.SendMessageRequest; import com.amazonaws.services.sqs.model.SendMessageResult; import com.google.common.util.concurrent.FutureCallback; @@ -37,7 +36,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -82,13 +80,6 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); - Map attributes = new HashMap<>(); - - attributes.put("headers", new MessageAttributeValue() - .withStringValue(gson.toJson(msg.getHeaders().getData())) - .withDataType("String")); - - sendMsgRequest.withMessageAttributes(attributes); sendMsgRequest.withMessageGroupId(msg.getKey().toString()); ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); diff --git a/pom.xml b/pom.xml index e385bff9ce..96f336c3a2 100755 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ 1.23 1.11.747 1.84.0 + 3.2.0 1.5.0 1.4.3 @@ -898,6 +899,11 @@ google-cloud-pubsub ${pubsub.client.version} + + com.microsoft.azure + azure-servicebus + ${azure-servicebus.version} + org.passay passay diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index a875d56782..37ac4c14c2 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -83,6 +83,11 @@ queue: ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 3eaae1ef32cf601ca06001e95a5f5f130c4d74a8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 8 Apr 2020 14:09:56 +0300 Subject: [PATCH 033/102] Statistics Implementation --- .../server/actors/app/AppActor.java | 35 +---- .../device/DeviceActorMessageProcessor.java | 4 +- .../actors/ruleChain/DefaultTbContext.java | 39 +++--- .../RemoteToRuleChainTellNextMsg.java | 48 ------- .../actors/ruleChain/RuleChainActor.java | 16 +-- .../RuleChainActorMessageProcessor.java | 34 +++-- .../ruleChain/RuleChainManagerActor.java | 68 ++++++++-- .../RuleNodeToRuleChainTellNextMsg.java | 1 + .../actors/shared/EntityActorsManager.java | 93 ------------- .../shared/rulechain/RuleChainManager.java | 59 -------- .../rulechain/SystemRuleChainManager.java | 48 ------- .../rulechain/TenantRuleChainManager.java | 53 -------- .../server/actors/stats/StatsActor.java | 1 - .../server/actors/tenant/TenantActor.java | 66 +++++---- .../controller/plugin/TbWebSocketHandler.java | 6 +- .../queue/DefaultTbCoreConsumerService.java | 20 +-- .../DefaultTbRuleEngineConsumerService.java | 54 +++++--- .../service/queue/TbMsgPackCallback.java | 78 +++++++++++ ...gPackCallback.java => TbPackCallback.java} | 21 +-- .../queue/TbRuleEngineConsumerStats.java | 119 +++++++++++++--- .../queue/TbTenantRuleEngineStats.java | 92 +++++++++++++ .../processing/AbstractConsumerService.java | 10 +- .../TbRuleEngineProcessingResult.java | 18 ++- ...TbRuleEngineProcessingStrategyFactory.java | 31 +++-- .../service/rpc/FromDeviceRpcResponse.java | 1 - .../rpc/ToDeviceRpcRequestActorMsg.java | 3 - .../state/DefaultDeviceStateService.java | 4 +- .../service/state/DeviceStateService.java | 4 +- .../DefaultRuleEngineStatisticsService.java | 127 ++++++++++++++++++ .../stats/RuleEngineStatisticsService.java | 12 +- .../DefaultSubscriptionManagerService.java | 12 +- .../DefaultTbLocalSubscriptionService.java | 8 +- .../SubscriptionManagerService.java | 10 +- .../TbLocalSubscriptionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 6 +- .../TelemetrySubscriptionService.java | 4 - .../service/telemetry/sub/Subscription.java | 80 ----------- .../BaseRuleChainTransactionService.java | 13 -- .../msg/TransportToDeviceActorMsgWrapper.java | 6 +- application/src/main/resources/logback.xml | 4 +- .../src/main/resources/thingsboard.yml | 6 +- ...AbstractRuleEngineFlowIntegrationTest.java | 2 - ...actRuleEngineLifecycleIntegrationTest.java | 2 - .../server/common/msg/MsgType.java | 5 - .../thingsboard/server/common/msg/TbMsg.java | 4 +- .../common/msg/cluster/ServerAddress.java | 47 ------- .../msg/queue/QueueToRuleEngineMsg.java | 1 + .../common/msg/queue/RuleEngineException.java | 38 ++++++ .../common/msg/queue/RuleNodeException.java | 58 ++++++++ .../server/common/msg/queue/TbCallback.java | 37 +++++ .../common/msg/queue/TbMsgCallback.java | 4 +- .../MultipleTbQueueTbMsgCallbackWrapper.java | 4 +- .../common/TbQueueTbMsgCallbackWrapper.java | 6 +- .../ConsistentHashPartitionService.java | 37 ++--- .../DefaultTbServiceInfoProvider.java | 13 ++ .../queue/discovery/PartitionService.java | 2 + .../discovery/TbServiceInfoProvider.java | 8 ++ .../settings/TbQueueRuleEngineSettings.java | 3 + common/queue/src/main/proto/queue.proto | 1 + .../api/RuleChainTransactionService.java | 3 - .../rule/engine/api/TbContext.java | 4 +- .../rule/engine/flow/TbAckNode.java | 53 ++++++++ .../rule/engine/flow/TbCheckpointNode.java | 57 ++++++++ .../flow/TbCheckpointNodeConfiguration.java | 24 ++-- .../rule/engine/rpc/TbSendRPCRequestNode.java | 3 +- .../src/main/resources/tb-mqtt-transport.yml | 38 +++++- 66 files changed, 1014 insertions(+), 758 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java rename application/src/main/java/org/thingsboard/server/service/queue/{MsgPackCallback.java => TbPackCallback.java} (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java => application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java (72%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java => rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java (53%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index c096238a8b..145ae10d06 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -24,10 +24,9 @@ import akka.actor.Terminated; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; +import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -37,16 +36,14 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; -public class AppActor extends RuleChainManagerActor { +public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; @@ -54,7 +51,7 @@ public class AppActor extends RuleChainManagerActor { private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext, new SystemRuleChainManager(systemContext)); + super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); } @@ -80,9 +77,6 @@ public class AppActor extends RuleChainManagerActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case SEND_TO_CLUSTER_MSG: - onPossibleClusterMsg((SendToClusterMsg) msg); - break; case PARTITION_CHANGE_MSG: broadcast(msg); break; @@ -98,7 +92,6 @@ public class AppActor extends RuleChainManagerActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -110,7 +103,6 @@ public class AppActor extends RuleChainManagerActor { private void initRuleChainsAndTenantActors() { log.info("Starting main system actor."); try { - initRuleChains(); if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); for (Tenant tenant : tenantIterator) { @@ -125,37 +117,22 @@ public class AppActor extends RuleChainManagerActor { } } - private void onPossibleClusterMsg(SendToClusterMsg msg) { - //TODO 2.5 -// Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); -// if (address.isPresent()) { - -// systemContext.getRpcService().tell( -// systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); -// } else { - self().tell(msg.getMsg(), ActorRef.noSender()); -// } - } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { -// this may be a notification about system entities created. -// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); + msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); } } - @Override protected void broadcast(Object msg) { - super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - target = getEntityActorRef(msg.getEntityId()); + log.warn("Message has system tenant id: {}", msg); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 307797f864..83d269ef6d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -56,7 +56,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; @@ -213,7 +213,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { TransportToDeviceActorMsg msg = wrapper.getMsg(); - TbMsgCallback callback = wrapper.getCallback(); + TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7178b94dfa..0e8033fac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -110,7 +110,7 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } @Override @@ -139,53 +139,61 @@ class DefaultTbContext implements TbContext { mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); } + @Override + public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); + } + @Override public void enqueueForTellNext(TbMsg tbMsg, String relationType) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); } @Override public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, null); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); } @Override public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); } - private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)) - .addAllRelationTypes(relationTypes) - .build(); - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + .addAllRelationTypes(relationTypes); + if (failureMessage != null) { + msg.setFailureMessage(failureMessage); + } + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg.build()), new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override @@ -207,7 +215,8 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), + msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } public void updateSelf(RuleNode self) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java deleted file mode 100644 index d062c5dc49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.ruleChain; - -import lombok.Data; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.aware.TenantAwareMsg; - -import java.io.Serializable; - -/** - * Created by ashvayka on 19.03.18. - */ -@Data -final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { - - private static final long serialVersionUID = 2459605482321657447L; - private final TenantId tenantId; - private final RuleChainId ruleChainId; - - public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { - super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); - this.tenantId = tenantId; - this.ruleChainId = ruleChainId; - } - - @Override - public MsgType getMsgType() { - return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 015e7a008b..027f4aac9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @@ -30,9 +31,9 @@ import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { - super(systemContext, tenantId, ruleChainId); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { + super(systemContext, tenantId, ruleChain.getId()); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, context().parent(), context().self())); } @@ -46,7 +47,6 @@ public class RuleChainActor extends ComponentActor> nodeRoutes; private final RuleChainService service; private final TbQueueProducer> producer; + private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChainId); + super(systemContext, tenantId, ruleChain.getId()); + this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); @@ -113,6 +117,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); for (RuleNode ruleNode : ruleNodeList) { @@ -194,7 +199,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes) { + private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set relationTypes, String failureMessage) { try { checkActive(); EntityId entityId = msg.getOriginator(); @@ -245,9 +250,14 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; - public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { + public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.ruleChainManager = ruleChainManager; + this.tenantId = tenantId; + this.actors = HashBiMap.create(); this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - ruleChainManager.init(this.context()); + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + RuleChainId ruleChainId = ruleChain.getId(); + log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); + visit(ruleChain, actorRef); + log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); + } + } + + protected void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + + public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId) { + return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); + } + + public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId, Function provider) { + return actors.computeIfAbsent(ruleChainId, eId -> { + RuleChain ruleChain = provider.apply(eId); + return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) + .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); + }); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - switch (entityId.getEntityType()) { - case RULE_CHAIN: - target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); - break; + if (entityId.getEntityType() == EntityType.RULE_CHAIN) { + target = getOrCreateActor(this.context(), (RuleChainId) entityId); } return target; } protected void broadcast(Object msg) { - ruleChainManager.broadcast(msg); + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } + + public ActorRef get(RuleChainId id) { + return actors.get(id); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java index 1fda91caf0..2bccc654e0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java @@ -34,6 +34,7 @@ class RuleNodeToRuleChainTellNextMsg implements TbActorMsg, Serializable { private final RuleNodeId originator; private final Set relationTypes; private final TbMsg msg; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java deleted file mode 100644 index e45f69ca9e..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared; - -import akka.actor.ActorContext; -import akka.actor.ActorRef; -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.common.data.SearchTextBased; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageDataIterable; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class EntityActorsManager> { - - protected final ActorSystemContext systemContext; - protected final BiMap actors; - - public EntityActorsManager(ActorSystemContext systemContext) { - this.systemContext = systemContext; - this.actors = HashBiMap.create(); - } - - protected abstract TenantId getTenantId(); - - protected abstract String getDispatcherName(); - - protected abstract Creator creator(T entityId); - - protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); - - public void init(ActorContext context) { - for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { - T entityId = (T) entity.getId(); - log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(context, entityId); - visit(entity, actorRef); - log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); - } - } - - public void visit(M entity, ActorRef actorRef) { - } - - public ActorRef getOrCreateActor(ActorContext context, T entityId) { - return actors.computeIfAbsent(entityId, eId -> - context.actorOf(Props.create(creator(eId)) - .withDispatcher(getDispatcherName()), eId.toString())); - } - - public void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); - } - - public void remove(T id) { - actors.remove(id); - } - - public ActorRef get(T id) { - return actors.get(id); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java deleted file mode 100644 index ef2b4282d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorRef; -import akka.japi.Creator; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.shared.EntityActorsManager; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.rule.RuleChainService; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class RuleChainManager extends EntityActorsManager { - - protected final RuleChainService service; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; - - public RuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - this.service = systemContext.getRuleChainService(); - } - - @Override - public Creator creator(RuleChainId entityId) { - return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); - } - - @Override - public void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java deleted file mode 100644 index f2a80a9e82..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.page.TextPageData; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.model.ModelConstants; - -import java.util.Collections; - -public class SystemRuleChainManager extends RuleChainManager { - - public SystemRuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> new TextPageData<>(Collections.emptyList(), link); - } - - @Override - protected TenantId getTenantId() { - return ModelConstants.SYSTEM_TENANT; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java deleted file mode 100644 index cfe862af49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorContext; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; - -public class TenantRuleChainManager extends RuleChainManager { - - private final TenantId tenantId; - - public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext); - this.tenantId = tenantId; - } - - @Override - public void init(ActorContext context) { - super.init(context); - } - - @Override - protected TenantId getTenantId() { - return tenantId; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> service.findTenantRuleChains(tenantId, link); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 1ef33467b2..717dcecc19 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 3a719ca857..e54b955419 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -30,7 +30,6 @@ import org.thingsboard.server.actors.device.DeviceActorCreator; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -43,6 +42,7 @@ import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; @@ -51,12 +51,12 @@ import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { - private final TenantId tenantId; private final BiMap deviceActors; + private boolean isRuleEngine; + private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); - this.tenantId = tenantId; + super(systemContext, tenantId); this.deviceActors = HashBiMap.create(); } @@ -69,7 +69,11 @@ public class TenantActor extends RuleChainManagerActor { public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - initRuleChains(); + isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + if (isRuleEngine) { + initRuleChains(); + } log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); @@ -115,7 +119,6 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -129,16 +132,19 @@ public class TenantActor extends RuleChainManagerActor { } private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + if (!isRuleEngine) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); + if (getRootChainActor() != null) { + getRootChainActor().tell(msg, self()); } else { - tbMsg.getCallback().onFailure(new RuntimeException("No Root Rule Chain available!")); + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); log.info("[{}] No Root Chain: {}", tenantId, msg); } } else { - ActorRef ruleChainActor = ruleChainManager.get(tbMsg.getRuleChainId()); + ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); if (ruleChainActor != null) { ruleChainActor.tell(msg, self()); } else { @@ -150,24 +156,29 @@ public class TenantActor extends RuleChainManagerActor { } private void onRuleChainMsg(RuleChainAwareMsg msg) { - ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { + if (!isCore) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - ruleChainManager.visit(ruleChain, target); + if (isRuleEngine) { + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + visit(ruleChain, target); + } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -214,15 +225,12 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { - @Override - public SupervisorStrategy.Directive apply(Throwable t) { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); - } + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); } }); diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index f7d956e146..6e33133522 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -92,10 +92,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -139,7 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 434f1d7eae..401262a243 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -120,8 +120,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToSubscriptionMgrMsg()) { @@ -182,7 +182,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbMsgCallback callback) { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCoreNotificationMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); @@ -200,7 +200,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) , proto.getResponse(), error); @@ -215,7 +215,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, - TbRuleEngineQueueFactory tbRuleEngineQueueFactory, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.statisticsService = statisticsService; this.ruleEngineSettings = ruleEngineSettings; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.factory = factory; @@ -92,7 +98,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< public void init() { super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumerConfigurations.putIfAbsent(configuration.getName(), configuration); consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); } } @@ -107,7 +115,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue))); + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); } @Override @@ -115,7 +123,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< consumers.values().forEach(TbQueueConsumer::unsubscribe); } - private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration) { + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { consumersExecutor.execute(() -> { while (!stopped) { try { @@ -123,7 +131,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getAckStrategy()); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getName(), configuration.getAckStrategy()); TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { @@ -137,21 +145,21 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } ConcurrentMap> successMap = new ConcurrentHashMap<>(); ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - + ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { - log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = new TbMsgPackCallback<>(id, tenantId, processingTimeoutLatch, allMap, successMap, failedMap, exceptionsMap); try { - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } - } catch (Throwable e) { - callback.onFailure(e); + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage())); } }); @@ -159,7 +167,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { timeout = true; } - decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap, exceptionsMap); + decision = strategy.analyze(result); + if (statsEnabled) { + stats.log(result, decision.isCommit()); + } } consumer.commit(); } catch (Exception e) { @@ -193,7 +205,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); @@ -219,18 +231,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< relationTypes = new HashSet<>(relationTypesList); } } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes); + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); actorContext.getAppActor().tell(msg, ActorRef.noSender()); - //TODO: 2.5 before release. -// if (statsEnabled) { -// stats.log(toDeviceActorMsg); -// } } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { - stats.printStats(); + long ts = System.currentTimeMillis(); + consumerStats.forEach((queue, stats) -> { + stats.printStats(); + statisticsService.reportQueueStats(ts, stats); + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java new file mode 100644 index 0000000000..0470db4cd7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class TbMsgPackCallback implements TbMsgCallback { + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap successMap; + private final ConcurrentMap failedMap; + private final UUID id; + private final TenantId tenantId; + private final ConcurrentMap firstExceptions; + + public TbMsgPackCallback(UUID id, TenantId tenantId, + CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap successMap, + ConcurrentMap failedMap, + ConcurrentMap firstExceptions) { + this.id = id; + this.tenantId = tenantId; + this.processingTimeoutLatch = processingTimeoutLatch; + this.ackMap = ackMap; + this.successMap = successMap; + this.failedMap = failedMap; + this.firstExceptions = firstExceptions; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + T msg = ackMap.remove(id); + if (msg != null) { + successMap.put(id, msg); + } + if (msg != null && ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } + + @Override + public void onFailure(RuleEngineException e) { + log.trace("[{}] ON FAILURE", id, e); + T msg = ackMap.remove(id); + if (msg != null) { + failedMap.put(id, msg); + firstExceptions.putIfAbsent(tenantId, e); + } + if (ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 72c5c10458..ba5a883ea0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -16,30 +16,26 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class MsgPackCallback implements TbMsgCallback { +public class TbPackCallback implements TbCallback { private final CountDownLatch processingTimeoutLatch; private final ConcurrentMap ackMap; - private final ConcurrentMap successMap; private final ConcurrentMap failedMap; private final UUID id; - public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap successMap, - ConcurrentMap failedMap) { + public TbPackCallback(UUID id, + CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { this.id = id; this.processingTimeoutLatch = processingTimeoutLatch; this.ackMap = ackMap; - this.successMap = successMap; this.failedMap = failedMap; } @@ -47,9 +43,6 @@ public class MsgPackCallback implements TbMsgCallback { public void onSuccess() { log.trace("[{}] ON SUCCESS", id); T msg = ackMap.remove(id); - if (msg != null) { - successMap.put(id, msg); - } if (msg != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } @@ -57,7 +50,7 @@ public class MsgPackCallback implements TbMsgCallback { @Override public void onFailure(Throwable t) { - log.trace("[{}] ON FAILURE", id); + log.trace("[{}] ON FAILURE", id, t); T msg = ackMap.remove(id); if (msg != null) { failedMap.put(id, msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 27792c5a61..22f4edbbba 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -15,38 +15,119 @@ */ package org.thingsboard.server.service.queue; +import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @Slf4j +@Data public class TbRuleEngineConsumerStats { - private final AtomicInteger totalCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); + public static final String TOTAL_MSGS = "totalMsgs"; + public static final String SUCCESSFUL_MSGS = "successfulMsgs"; + public static final String TMP_TIMEOUT = "tmpTimeout"; + public static final String TMP_FAILED = "tmpFailed"; + public static final String TIMEOUT_MSGS = "timeoutMsgs"; + public static final String FAILED_MSGS = "failedMsgs"; + public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; + public static final String FAILED_ITERATIONS = "failedIterations"; - public void log(TransportProtos.TransportToRuleEngineMsg msg) { - totalCounter.incrementAndGet(); - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger successIterationsCounter = new AtomicInteger(0); + private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); + + private final String queueName; + + public TbRuleEngineConsumerStats(String queueName) { + this.queueName = queueName; + counters.put(TOTAL_MSGS, totalMsgCounter); + counters.put(SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(FAILED_MSGS, failedMsgCounter); + + counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TMP_FAILED, tmpFailedMsgCounter); + counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); + counters.put(FAILED_ITERATIONS, failedIterationsCounter); + } + + public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { + int success = msg.getSuccessMap().size(); + int pending = msg.getPendingMap().size(); + int failed = msg.getFailureMap().size(); + totalMsgCounter.addAndGet(success + pending + failed); + successMsgCounter.addAndGet(success); + msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); + if (finalIterationForPack) { + if (pending > 0 || failed > 0) { + timeoutMsgCounter.addAndGet(pending); + failedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); + } + if (failed > 0) { + msg.getFailureMap().values().forEach(m -> getTenantStats(m).logFailed()); + } + failedIterationsCounter.incrementAndGet(); + } else { + successIterationsCounter.incrementAndGet(); + } + } else { + failedIterationsCounter.incrementAndGet(); + tmpTimeoutMsgCounter.addAndGet(pending); + tmpFailedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); + } + if (failed > 0) { + msg.getFailureMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + } } + msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); + } + + private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { + ToRuleEngineMsg reMsg = m.getValue(); + return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); } public void printStats() { - int total = totalCounter.getAndSet(0); + int total = totalMsgCounter.get(); if (total > 0) { - log.info("Transport total [{}] telemetry [{}] attributes [{}] toServerRpc [{}]", - total, postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), toServerRPCCallRequestCounter.getAndSet(0)); + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("] "); + }); + log.info("[{}] Stats: {}", queueName, stats); } } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + tenantStats.clear(); + tenantExceptions.clear(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java new file mode 100644 index 0000000000..fb36f5064b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbTenantRuleEngineStats { + + private final UUID tenantId; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + + public TbTenantRuleEngineStats(UUID tenantId) { + this.tenantId = tenantId; + counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); + counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); + + counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); + } + + public void logSuccess() { + totalMsgCounter.incrementAndGet(); + successMsgCounter.incrementAndGet(); + } + + public void logFailed() { + totalMsgCounter.incrementAndGet(); + failedMsgCounter.incrementAndGet(); + } + + public void logTimeout() { + totalMsgCounter.incrementAndGet(); + timeoutMsgCounter.incrementAndGet(); + } + + public void logTmpFailed() { + totalMsgCounter.incrementAndGet(); + tmpFailedMsgCounter.incrementAndGet(); + } + + public void logTmpTimeout() { + totalMsgCounter.incrementAndGet(); + tmpTimeoutMsgCounter.incrementAndGet(); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("]"); + }); + log.info("[{}] Stats: {}", tenantId, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0df0517232..a238e34337 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -22,12 +22,12 @@ import org.springframework.context.event.EventListener; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.MsgPackCallback; +import org.thingsboard.server.service.queue.TbPackCallback; import javax.annotation.PreDestroy; import java.util.List; @@ -95,8 +95,8 @@ public abstract class AbstractConsumerService> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - log.info("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); try { handleNotification(id, msg, callback); } catch (Throwable e) { @@ -124,7 +124,7 @@ public abstract class AbstractConsumerService msg, TbMsgCallback callback) throws Exception; + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; @PreDestroy public void destroy() { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index c17cf54bca..e67936e294 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.queue.processing; import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -25,24 +27,28 @@ import java.util.concurrent.ConcurrentMap; public class TbRuleEngineProcessingResult { @Getter - private boolean success; + private final boolean success; @Getter - private boolean timeout; + private final boolean timeout; @Getter - private ConcurrentMap> pendingMap; + private final ConcurrentMap> pendingMap; @Getter - private ConcurrentMap> successMap; + private final ConcurrentMap> successMap; @Getter - private ConcurrentMap> failureMap; + private final ConcurrentMap> failureMap; + @Getter + private final ConcurrentMap exceptionsMap; public TbRuleEngineProcessingResult(boolean timeout, ConcurrentMap> pendingMap, ConcurrentMap> successMap, - ConcurrentMap> failureMap) { + ConcurrentMap> failureMap, + ConcurrentMap exceptionsMap) { this.timeout = timeout; this.pendingMap = pendingMap; this.successMap = successMap; this.failureMap = failureMap; + this.exceptionsMap = exceptionsMap; this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 85a44794d8..1096bfbf7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -32,24 +32,25 @@ import java.util.concurrent.TimeUnit; @Slf4j public class TbRuleEngineProcessingStrategyFactory { - public TbRuleEngineProcessingStrategy newInstance(TbRuleEngineQueueAckStrategyConfiguration configuration) { + public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { switch (configuration.getType()) { case "SKIP_ALL": - return new SkipStrategy(); + return new SkipStrategy(name); case "RETRY_ALL": - return new RetryStrategy(true, true, true, configuration); + return new RetryStrategy(name, true, true, true, configuration); case "RETRY_FAILED": - return new RetryStrategy(false, true, false, configuration); + return new RetryStrategy(name, false, true, false, configuration); case "RETRY_TIMED_OUT": - return new RetryStrategy(false, false, true, configuration); + return new RetryStrategy(name, false, false, true, configuration); case "RETRY_FAILED_AND_TIMED_OUT": - return new RetryStrategy(false, true, true, configuration); + return new RetryStrategy(name, false, true, true, configuration); default: throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); } } private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; private final boolean retrySuccessful; private final boolean retryFailed; private final boolean retryTimeout; @@ -60,7 +61,8 @@ public class TbRuleEngineProcessingStrategyFactory { private int initialTotalCount; private int retryCount; - public RetryStrategy(boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + this.queueName = queueName; this.retrySuccessful = retrySuccessful; this.retryFailed = retryFailed; this.retryTimeout = retryTimeout; @@ -80,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory { retryCount++; double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { - log.info("Skip reprocess of the rule engine pack due to max retries"); + log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("Skip reprocess of the rule engine pack due to max allowed failure percentage"); + log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); return new TbRuleEngineProcessingDecision(true, null); } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); @@ -96,8 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory { if (retrySuccessful) { result.getSuccessMap().forEach(toReprocess::put); } - log.info("Going to reprocess {} messages", toReprocess.size()); - //TODO: 2.5 Log most popular rule nodes by error count; + log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); } @@ -116,9 +117,15 @@ public class TbRuleEngineProcessingStrategyFactory { private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; + + public SkipStrategy(String name) { + this.queueName = name; + } + @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("Skip reprocess of the rule engine pack"); + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailureMap().size(), result.getPendingMap().size()); return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index 78ea0c5660..c1e5e2e038 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,7 +19,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index 3a1ea599e6..3cfe9bc1f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,11 +22,8 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import java.util.Optional; - /** * Created by ashvayka on 16.04.18. */ diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 32980c0064..b4c5719844 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -55,7 +55,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -242,7 +242,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { try { TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index c7aadda963..16c699b38a 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; /** * Created by ashvayka on 01.05.18. @@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to persist statistics", t); + } + }; + + private final TbServiceInfoProvider serviceInfoProvider; + private final TelemetrySubscriptionService tsService; + private final Lock lock = new ReentrantLock(); + private final AssetService assetService; + private final ConcurrentMap tenantQueueAssets; + + public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { + this.tsService = tsService; + this.serviceInfoProvider = serviceInfoProvider; + this.assetService = assetService; + this.tenantQueueAssets = new ConcurrentHashMap<>(); + } + + @Override + public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { + String queueName = ruleEngineStats.getQueueName(); + ruleEngineStats.getTenantStats().forEach((id, stats) -> { + TenantId tenantId = new TenantId(id); + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + }); + ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { + TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + }); + ruleEngineStats.reset(); + } + + private AssetId getServiceAssetId(TenantId tenantId, String queueName) { + TenantQueueKey key = new TenantQueueKey(tenantId, queueName); + AssetId assetId = tenantQueueAssets.get(key); + if (assetId == null) { + lock.lock(); + try { + assetId = tenantQueueAssets.get(key); + if (assetId == null) { + Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); + if (asset == null) { + asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); + asset.setType("TbServiceQueue"); + asset = assetService.saveAsset(asset); + } + assetId = asset.getId(); + tenantQueueAssets.put(key, assetId); + } + } finally { + lock.unlock(); + } + } + return assetId; + } + + @Data + private static class TenantQueueKey { + private final TenantId tenantId; + private final String queueName; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java rename to application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java index 4b00284c34..88573844c9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.stats; -/** - * Created by ashvayka on 23.09.18. - */ -public enum ServerType { - CORE +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; + +public interface RuleEngineStatisticsService { + + void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index b6955e40a9..0d151c3f46 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -123,7 +123,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void addSubscription(TbSubscription subscription, TbMsgCallback callback) { + public void addSubscription(TbSubscription subscription, TbCallback callback) { log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); @@ -151,7 +151,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback) { + public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); if (sessionSubscriptions != null) { @@ -189,7 +189,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { @@ -213,7 +213,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback) { + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { @@ -261,7 +261,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { if (serviceId.equals(s.getServiceId())) { SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); } else { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index e4d0d838ca..12c5e206e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; @@ -135,7 +135,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (currentPartitions.contains(tpi)) { // Subscription is managed on the same server; if (pushToLocalService) { - subscriptionManagerService.addSubscription(subscription, TbMsgCallback.EMPTY); + subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); } } else { // Push to the queue; @@ -145,7 +145,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } @Override - public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback) { + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { TbSubscription subscription = subscriptionsBySessionId .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); if (subscription != null) { @@ -177,7 +177,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); if (currentPartitions.contains(tpi)) { // Subscription is managed on the same server; - subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbMsgCallback.EMPTY); + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index aa1824c855..e20b707348 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -21,18 +21,18 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.List; public interface SubscriptionManagerService extends ApplicationListener { - void addSubscription(TbSubscription subscription, TbMsgCallback callback); + void addSubscription(TbSubscription subscription, TbCallback callback); - void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback); + void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); - void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); - void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index b6bee14a32..58fba24250 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -17,7 +17,7 @@ package org.thingsboard.server.service.subscription; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; public interface TbLocalSubscriptionService { @@ -28,7 +28,7 @@ public interface TbLocalSubscriptionService { void cancelAllSessionSubscriptions(String sessionId); - void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback); + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); void onApplicationEvent(PartitionChangeEvent event); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 77b0967966..b3c728a25b 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -166,7 +166,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); } else { log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } @@ -180,7 +180,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); } else { log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index e905559516..d652a3f1c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -17,11 +17,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java deleted file mode 100644 index 7f7db9430d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.telemetry.sub; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.TelemetryFeature; - -import java.util.Map; - -@Data -@AllArgsConstructor -public class Subscription { - - private final SubscriptionState sub; - private final boolean local; - private ServerAddress server; - private long startTime; - private long endTime; - - public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { - this(sub, local, server, 0L, 0L); - } - - public String getWsSessionId() { - return getSub().getWsSessionId(); - } - - public int getSubscriptionId() { - return getSub().getSubscriptionId(); - } - - public EntityId getEntityId() { - return getSub().getEntityId(); - } - - public TelemetryFeature getType() { - return getSub().getType(); - } - - public String getScope() { - return getSub().getScope(); - } - - public boolean isAllKeys() { - return getSub().isAllKeys(); - } - - public Map getKeyStates() { - return getSub().getKeyStates(); - } - - public void setKeyState(String key, long ts) { - getSub().getKeyStates().put(key, ts); - } - - @Override - public String toString() { - return "Subscription{" + - "sub=" + sub + - ", local=" + local + - ", server=" + server + - '}'; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 783dd5181a..477f115a96 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -22,7 +22,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -114,13 +113,6 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ // } } - @Override - public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data, null), msg -> { - }, error -> { - }); - } - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { queue.offer(transactionTask); timeoutQueue.offer(transactionTask); @@ -230,9 +222,4 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ callbackExecutor.executeAsync(task); } - private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { - log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - //TODO 2.5 -// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 16e0dd67f2..ba4e7baab1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -37,9 +37,9 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; - private final TbMsgCallback callback; + private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbMsgCallback callback) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { this.msg = msg; this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 3d47db432f..8389fa5d71 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,8 +27,8 @@ - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 04c9205a12..79efacba97 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -561,7 +561,7 @@ queue: poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - @@ -577,7 +577,7 @@ queue: failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "HighPriority" + name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" @@ -594,7 +594,7 @@ queue: poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 5ecb5fb415..7fe63827eb 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -35,8 +35,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index bc4db3b535..78129917ef 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index dec6f6b50b..47e09b812f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -33,11 +33,6 @@ public enum MsgType { APP_INIT_MSG, - /** - * All messages, could be send to cluster - */ - SEND_TO_CLUSTER_MSG, - /** * ADDED/UPDATED/DELETED events for main entities. * diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7230804773..84110ed163 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,7 +17,6 @@ package org.thingsboard.server.common.msg; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -29,7 +28,6 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.UUID; /** diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java deleted file mode 100644 index 80674f9bce..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.cluster; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - * @author Andrew Shvayka - */ -@Data -@EqualsAndHashCode -public class ServerAddress implements Comparable, Serializable { - - private final String host; - private final int port; - private final ServerType serverType; - - @Override - public int compareTo(ServerAddress o) { - int result = this.host.compareTo(o.host); - if (result == 0) { - result = this.port - o.port; - } - return result; - } - - @Override - public String toString() { - return '[' + host + ':' + port + ']'; - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index 49b30a5992..fca1c51995 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -33,6 +33,7 @@ public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; private final Set relationTypes; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java new file mode 100644 index 0000000000..dd720251bb --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuleEngineException extends Exception { + protected static final ObjectMapper mapper = new ObjectMapper(); + + public RuleEngineException(String message) { + super(message != null ? message : "Unknown"); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java new file mode 100644 index 0000000000..3f437dd1d7 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.rule.RuleNode; + +@Slf4j +public class RuleNodeException extends RuleEngineException { + @Getter + private final String ruleChainName; + @Getter + private final String ruleNodeName; + @Getter + private final RuleChainId ruleChainId; + @Getter + private final RuleNodeId ruleNodeId; + + public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { + super(message); + this.ruleChainName = ruleChainName; + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode() + .put("ruleNodeId", ruleNodeId.toString()) + .put("ruleChainId", ruleChainId.toString()) + .put("ruleNodeName", ruleNodeName) + .put("ruleChainName", ruleChainName) + .put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java new file mode 100644 index 0000000000..4c25f27a3e --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +public interface TbCallback { + + TbCallback EMPTY = new TbCallback() { + + @Override + public void onSuccess() { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + + void onSuccess(); + + void onFailure(Throwable t); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 4f0e383fe9..9e8d5ae6b8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -25,13 +25,13 @@ public interface TbMsgCallback { } @Override - public void onFailure(Throwable t) { + public void onFailure(RuleEngineException e) { } }; void onSuccess(); - void onFailure(Throwable t); + void onFailure(RuleEngineException e); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java index 552eca84a0..f860d8dcc9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.common; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; @@ -40,6 +42,6 @@ public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { @Override public void onFailure(Throwable t) { - tbMsgCallback.onFailure(t); + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java index c372f64b3d..8bd9c3516b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.queue.common; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import java.util.concurrent.atomic.AtomicInteger; - public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { private final TbMsgCallback tbMsgCallback; @@ -36,6 +36,6 @@ public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { @Override public void onFailure(Throwable t) { - tbMsgCallback.onFailure(t); + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 1337a1cc74..3558374af5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -86,24 +86,6 @@ public class ConsistentHashPartitionService implements PartitionService { partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); } -// public Set getCurrentPartitions(ServiceType serviceType) { -// ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); -// TenantId tenantId = getSystemOrIsolatedTenantId(currentService); -// ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceType, tenantId); -// List partitions = myPartitions.get(serviceQueueKey); -// Set topicPartitions = new LinkedHashSet<>(); -// for (Integer partition : partitions) { -// TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); -// tpi.topic(partitionTopics.get(serviceType)); -// tpi.partition(partition); -// if (!tenantId.isNullUid()) { -// tpi.tenantId(tenantId); -// } -// topicPartitions.add(tpi.build()); -// } -// return topicPartitions; -// } - @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return resolve(new ServiceQueue(serviceType), tenantId, entityId); @@ -131,15 +113,7 @@ public class ConsistentHashPartitionService implements PartitionService { Map> circles = new HashMap<>(); addNode(circles, currentService); for (ServiceInfo other : otherServices) { - TenantId tenantId = getSystemOrIsolatedTenantId(other); addNode(circles, other); - if (!tenantId.isNullUid()) { - isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); - for (String serviceType : other.getServiceTypesList()) { - isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); - } - - } } ConcurrentMap> oldPartitions = myPartitions; TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); @@ -214,6 +188,11 @@ public class ConsistentHashPartitionService implements PartitionService { } } + @Override + public Set getIsolatedTenants(ServiceType serviceType) { + throw new RuntimeException("Not Implemented!"); + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { @@ -280,6 +259,12 @@ public class ConsistentHashPartitionService implements PartitionService { private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); + if (!tenantId.isNullUid()) { + isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); + for (String serviceType : instance.getServiceTypesList()) { + isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); + } + } for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 15cf33bb82..1c60a03cdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -34,6 +34,7 @@ import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -58,6 +59,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { private List serviceTypes; private ServiceInfo serviceInfo; + private TenantId isolatedTenant; @PostConstruct public void init() { @@ -80,6 +82,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { UUID tenantId; if (!StringUtils.isEmpty(tenantIdStr)) { tenantId = UUID.fromString(tenantIdStr); + isolatedTenant = new TenantId(tenantId); } else { tenantId = TenantId.NULL_UUID; } @@ -103,4 +106,14 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { public ServiceInfo getServiceInfo() { return serviceInfo; } + + @Override + public boolean isService(ServiceType serviceType) { + return serviceTypes.contains(serviceType); + } + + @Override + public Optional getIsolatedTenant() { + return Optional.ofNullable(isolatedTenant); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index e3b3e76b80..7c6900d8a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -47,6 +47,8 @@ public interface PartitionService { */ Set getAllServiceIds(ServiceType serviceType); + Set getIsolatedTenants(ServiceType serviceType); + /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 053dd644c5..fcc7c7a60d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,12 +15,20 @@ */ package org.thingsboard.server.queue.discovery; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Optional; + public interface TbServiceInfoProvider { String getServiceId(); ServiceInfo getServiceInfo(); + boolean isService(ServiceType serviceType); + + Optional getIsolatedTenant(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index 48eeb5ba2a..95540798fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,7 +17,9 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @@ -26,6 +28,7 @@ import java.util.List; @Slf4j @Data +@EnableAutoConfiguration @Configuration @ConfigurationProperties(prefix = "queue.rule-engine") public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 55d376b63b..e36c4e36a9 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -388,6 +388,7 @@ message ToRuleEngineMsg { int64 tenantIdLSB = 2; bytes tbMsg = 3; repeated string relationTypes = 4; + string failureMessage = 5; } message ToRuleEngineNotificationMsg { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java index bb0210b1ad..1bb56adb1d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java @@ -16,7 +16,6 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.function.Consumer; @@ -26,6 +25,4 @@ public interface RuleChainTransactionService { void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); - } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 72a552cb4f..4299aeb867 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,10 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -116,6 +114,8 @@ public interface TbContext { */ void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + void enqueueForTellFailure(TbMsg msg, String failureMessage); + void enqueueForTellNext(TbMsg msg, String relationType); void enqueueForTellNext(TbMsg msg, Set relationTypes); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java new file mode 100644 index 0000000000..9351e970ed --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "acknowledge", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Acknowledges the incoming message", + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.") + +public class TbAckNode implements TbNode { + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.ack(msg); + ctx.tellSuccess(msg); + } + + @Override + public void destroy() { + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java new file mode 100644 index 0000000000..0364251076 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import static org.thingsboard.common.util.DonAsynchron.withCallback; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "checkpoint", + configClazz = TbCheckpointNodeConfiguration.class, + nodeDescription = "transfers the message to another queue", + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.") + +public class TbCheckpointNode implements TbNode { + + private TbCheckpointNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); + } + + @Override + public void destroy() { + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java similarity index 53% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java index 3326c33aa3..eb5b8e29d9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java @@ -13,28 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.rule.engine.flow; import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.rule.engine.api.NodeConfiguration; @Data -public class SendToClusterMsg implements TbActorMsg { - - private TbActorMsg msg; - private EntityId entityId; - - public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { - this.entityId = entityId; - this.msg = msg; - } +public class TbCheckpointNodeConfiguration implements NodeConfiguration { + private String queueName; @Override - public MsgType getMsgType() { - return MsgType.SEND_TO_CLUSTER_MSG; + public TbCheckpointNodeConfiguration defaultConfiguration() { + TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); + configuration.setQueueName("HighPriority"); + return configuration; } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 88b060fe5c..ad1d399e76 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -116,8 +116,7 @@ public class TbSendRPCRequestNode implements TbNode { ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.enqueueForTellNext(next, TbRelationTypes.FAILURE); + ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); } }); ctx.ack(msg); diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index a875d56782..573fe6d5df 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -102,20 +102,44 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - rule_engine: + rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: # TODO 2.5: specify correct ENV variable names. + - + name: "Main" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - + name: "HighPriority" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; transport: notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: - type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + type: "${TB_SERVICE_TYPE:tb-transport}" # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file From 5bebf098ab0dbc3b7ef5cebeb6db549cde3cc900 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 13:03:30 +0300 Subject: [PATCH 034/102] Ability to reset blacklisted functions --- .../script/AbstractJsInvokeService.java | 48 +++++++++++++++---- .../AbstractNashornJsInvokeService.java | 4 +- .../script/NashornJsInvokeService.java | 10 ++++ .../service/script/RemoteJsInvokeService.java | 11 +++++ .../src/main/resources/thingsboard.yml | 12 +++-- .../script/TestNashornJsInvokeService.java | 5 ++ 6 files changed, 73 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 31827f542e..daa74d013f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,25 +78,53 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); + protected abstract long getMaxBlacklistDuration(); + protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - switch (scriptType) { - case RULE_NODE_SCRIPT: - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); - default: - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); + if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); } + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - if (blackListedFunctions.containsKey(scriptId)) { - AtomicInteger errorCount = blackListedFunctions.get(scriptId); - return errorCount.get() >= getMaxErrors(); + BlackListInfo errorCount = blackListedFunctions.get(scriptId); + if (errorCount != null) { + if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { + blackListedFunctions.remove(scriptId); + return false; + } else { + return errorCount.get() >= getMaxErrors(); + } } else { return false; } } + + private class BlackListInfo { + private final AtomicInteger counter; + private long expirationTime; + + private BlackListInfo() { + this.counter = new AtomicInteger(0); + } + + public int get() { + return counter.get(); + } + + public int incrementAndGet() { + int result = counter.incrementAndGet(); + expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); + return result; + } + + public long getExpirationTime() { + return expirationTime; + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 49a6304ebb..a9acfa4f42 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 909565609c..66a14cc827 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; + @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -37,6 +39,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; + @Value("${js.local.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -56,4 +61,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 1b5332bcfc..b0554decea 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -36,6 +36,7 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -66,6 +67,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Value("${js.remote.max_errors}") private int maxErrors; + @Value("${js.remote.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -205,6 +209,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { + onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -216,6 +221,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { + onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -245,4 +251,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e4a234fdb9..f08d2baf46 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -425,11 +425,13 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" @@ -449,6 +451,8 @@ js: response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" @@ -573,8 +577,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - - - name: "Main" + - name: "Main" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" @@ -585,8 +588,7 @@ queue: retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" + - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 1065fbf6d3..8a89de5dff 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,4 +49,9 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return 100000; + } } From ec4e2c036f8e43cb1fdf7ee96f1f044f64a99598 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 13:09:54 +0300 Subject: [PATCH 035/102] No more failures on missing RE queue --- .../queue/discovery/ConsistentHashPartitionService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index d4e4d87f69..0552aa05c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -99,7 +99,14 @@ public class ConsistentHashPartitionService implements PartitionService { int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - int partition = Math.abs(hash % partitionSizes.get(serviceQueue)); + Integer partitionSize = partitionSizes.get(serviceQueue); + int partition; + if (partitionSize != null) { + partition = Math.abs(hash % partitionSize); + } else { + //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. + partition = 0; + } boolean isolatedTenant = isIsolated(serviceQueue, tenantId); TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); From ca193239bab7ba4331a61bd9e061f0f8998bc42c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 18:33:44 +0300 Subject: [PATCH 036/102] RE Submit Strategies --- .../server/actors/ActorSystemContext.java | 7 +- .../queue/DefaultTbCoreConsumerService.java | 6 +- .../DefaultTbRuleEngineConsumerService.java | 77 ++++++++----- .../queue/ProcessingAttemptContext.java | 89 +++++++++++++++ .../service/queue/TbMsgPackCallback.java | 38 +----- .../queue/TbRuleEngineConsumerStats.java | 7 +- .../AbstractTbRuleEngineSubmitStrategy.java | 71 ++++++++++++ .../BatchTbRuleEngineSubmitStrategy.java | 86 ++++++++++++++ .../BurstTbRuleEngineSubmitStrategy.java | 50 ++++++++ .../service/queue/processing/IdMsgPair.java | 31 +++++ ...lByEntityIdTbRuleEngineSubmitStrategy.java | 108 ++++++++++++++++++ ...riginatorIdTbRuleEngineSubmitStrategy.java | 44 +++++++ ...lByTenantIdTbRuleEngineSubmitStrategy.java | 35 ++++++ .../SequentialTbRuleEngineSubmitStrategy.java | 73 ++++++++++++ .../TbRuleEngineProcessingResult.java | 38 +++--- ...TbRuleEngineProcessingStrategyFactory.java | 10 +- .../TbRuleEngineSubmitStrategy.java | 39 +++++++ .../TbRuleEngineSubmitStrategyFactory.java | 43 +++++++ .../rpc/DefaultTbCoreDeviceRpcService.java | 2 +- .../src/main/resources/thingsboard.yml | 46 +++++--- ...leEngineQueueAckStrategyConfiguration.java | 4 - .../TbRuleEngineQueueConfiguration.java | 3 +- ...ngineQueueSubmitStrategyConfiguration.java | 26 +++++ 23 files changed, 814 insertions(+), 119 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c266407707..d31cb62b9b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -334,7 +335,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Getter @Setter private ActorRef appActor; @@ -361,6 +361,8 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } + + public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -535,4 +537,7 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } + public void tell(TbActorMsg tbActorMsg, ActorRef sender) { + appActor.tell(tbActorMsg, sender); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 401262a243..263e6eab24 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -137,7 +137,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } @@ -194,7 +194,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } @@ -259,7 +259,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); + private ExecutorService submitExecutor; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, + TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + TbQueueRuleEngineSettings ruleEngineSettings, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); this.statisticsService = statisticsService; this.ruleEngineSettings = ruleEngineSettings; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.factory = factory; + this.submitStrategyFactory = submitStrategyFactory; + this.processingStrategyFactory = processingStrategyFactory; } @PostConstruct @@ -102,6 +111,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); } + submitExecutor = Executors.newSingleThreadExecutor(); + } + + @PreDestroy + public void stop() { + if (submitExecutor != null) { + submitExecutor.shutdownNow(); + } } @Override @@ -131,27 +148,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getName(), configuration.getAckStrategy()); - TbRuleEngineProcessingDecision decision = null; - boolean firstAttempt = true; - while (!stopped && (firstAttempt || !decision.isCommit())) { - ConcurrentMap> allMap; - if (firstAttempt) { - allMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - firstAttempt = false; - } else { - allMap = decision.getReprocessMap(); - } - ConcurrentMap> successMap = new ConcurrentHashMap<>(); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - allMap.forEach((id, msg) -> { - log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); + TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); + + submitStrategy.init(msgs); + + while (!stopped) { + ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { + log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = new TbMsgPackCallback<>(id, tenantId, processingTimeoutLatch, allMap, successMap, failedMap, exceptionsMap); + TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); try { if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); @@ -161,17 +169,24 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } catch (Exception e) { callback.onFailure(new RuleEngineException(e.getMessage())); } - }); + })); boolean timeout = false; - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { timeout = true; } - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap, exceptionsMap); - decision = strategy.analyze(result); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); if (statsEnabled) { stats.log(result, decision.isCommit()); } + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } } consumer.commit(); } catch (Exception e) { @@ -211,7 +226,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } else { @@ -232,7 +247,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.getAppActor().tell(msg, ActorRef.noSender()); + actorContext.tell(msg, ActorRef.noSender()); } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java new file mode 100644 index 0000000000..2073ff8c21 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ProcessingAttemptContext { + + private final TbRuleEngineSubmitStrategy submitStrategy; + + private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + @Getter + private final ConcurrentMap> pendingMap; + @Getter + private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); + + public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + this.submitStrategy = submitStrategy; + this.pendingMap = submitStrategy.getPendingMap(); + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + TbProtoQueueMsg msg; + boolean empty = false; + synchronized (pendingMap) { + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingMap.isEmpty(); + } + } + if (msg != null) { + successMap.put(id, msg); + } + submitStrategy.onSuccess(id); + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { + TbProtoQueueMsg msg; + boolean empty = false; + synchronized (pendingMap) { + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingMap.isEmpty(); + } + } + if (msg != null) { + failedMap.put(id, msg); + exceptionsMap.putIfAbsent(tenantId, e); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java index 0470db4cd7..2a6b6a658d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -27,52 +27,26 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class TbMsgPackCallback implements TbMsgCallback { - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap successMap; - private final ConcurrentMap failedMap; +public class TbMsgPackCallback implements TbMsgCallback { private final UUID id; private final TenantId tenantId; - private final ConcurrentMap firstExceptions; + private final ProcessingAttemptContext ctx; - public TbMsgPackCallback(UUID id, TenantId tenantId, - CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap successMap, - ConcurrentMap failedMap, - ConcurrentMap firstExceptions) { + public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { this.id = id; this.tenantId = tenantId; - this.processingTimeoutLatch = processingTimeoutLatch; - this.ackMap = ackMap; - this.successMap = successMap; - this.failedMap = failedMap; - this.firstExceptions = firstExceptions; + this.ctx = ctx; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - T msg = ackMap.remove(id); - if (msg != null) { - successMap.put(id, msg); - } - if (msg != null && ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onSuccess(id); } @Override public void onFailure(RuleEngineException e) { log.trace("[{}] ON FAILURE", id, e); - T msg = ackMap.remove(id); - if (msg != null) { - failedMap.put(id, msg); - firstExceptions.putIfAbsent(tenantId, e); - } - if (ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onFailure(tenantId, id, e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 22f4edbbba..40017d2b40 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; @@ -77,7 +76,7 @@ public class TbRuleEngineConsumerStats { public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { int success = msg.getSuccessMap().size(); int pending = msg.getPendingMap().size(); - int failed = msg.getFailureMap().size(); + int failed = msg.getFailedMap().size(); totalMsgCounter.addAndGet(success + pending + failed); successMsgCounter.addAndGet(success); msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); @@ -89,7 +88,7 @@ public class TbRuleEngineConsumerStats { msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); } if (failed > 0) { - msg.getFailureMap().values().forEach(m -> getTenantStats(m).logFailed()); + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); } failedIterationsCounter.incrementAndGet(); } else { @@ -103,7 +102,7 @@ public class TbRuleEngineConsumerStats { msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); } if (failed > 0) { - msg.getFailureMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); } } msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..bef733ec22 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { + + protected final String queueName; + protected List orderedMsgList; + private volatile boolean stopped; + + public AbstractTbRuleEngineSubmitStrategy(String queueName) { + this.queueName = queueName; + } + + protected abstract void doOnSuccess(UUID id); + + @Override + public void init(List> msgs) { + orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); + } + + @Override + public ConcurrentMap> getPendingMap() { + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); + for (IdMsgPair pair : orderedMsgList) { + if (reprocessMap.containsKey(pair.uuid)) { + newOrderedMsgList.add(pair); + } + } + orderedMsgList = newOrderedMsgList; + } + + @Override + public void onSuccess(UUID id) { + if (!stopped) { + doOnSuccess(id); + } + } + + @Override + public void stop() { + stopped = true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..d0b1f7f99a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final int batchSize; + private final AtomicInteger packIdx = new AtomicInteger(0); + private final Map> pendingPack = new LinkedHashMap<>(); + private volatile BiConsumer> msgConsumer; + + public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { + super(queueName); + this.batchSize = batchSize; + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + packIdx.set(0); + } + + @Override + protected void doOnSuccess(UUID id) { + boolean endOfPendingPack; + synchronized (pendingPack) { + TbProtoQueueMsg msg = pendingPack.remove(id); + endOfPendingPack = msg != null && pendingPack.isEmpty(); + } + if (endOfPendingPack) { + packIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int startIdx = Math.min(packIdx.get() * batchSize, listSize); + int endIdx = Math.min(startIdx + batchSize, listSize); + synchronized (pendingPack) { + pendingPack.clear(); + for (int i = startIdx; i < endIdx; i++) { + IdMsgPair pair = orderedMsgList.get(i); + pendingPack.put(pair.uuid, pair.msg); + } + } + int submitSize = pendingPack.size(); + if (log.isInfoEnabled() && submitSize > 0) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + } + pendingPack.forEach(msgConsumer); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ffd1dd49d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + public BurstTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + } + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + } + + @Override + protected void doOnSuccess(UUID id) { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java new file mode 100644 index 0000000000..2b2c203ec5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; + +public class IdMsgPair { + final UUID uuid; + final TbProtoQueueMsg msg; + + public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { + this.uuid = uuid; + this.msg = msg; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ae5993cb1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,108 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private volatile BiConsumer> msgConsumer; + private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); + + public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void init(List> msgs) { + super.init(msgs); + initMaps(); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + entityIdToListMap.forEach((entityId, queue) -> { + IdMsgPair msg = queue.peek(); + if (msg != null) { + msgConsumer.accept(msg.uuid, msg.msg); + } + }); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + initMaps(); + } + + @Override + protected void doOnSuccess(UUID id) { + EntityId entityId = msgToEntityIdMap.get(id); + if (entityId != null) { + Queue queue = entityIdToListMap.get(entityId); + if (queue != null) { + IdMsgPair next = null; + synchronized (queue) { + IdMsgPair expected = queue.peek(); + if (expected != null && expected.uuid.equals(id)) { + queue.poll(); + next = queue.peek(); + } + } + if (next != null) { + msgConsumer.accept(next.uuid, next.msg); + } + } + } + } + + private void initMaps() { + msgToEntityIdMap.clear(); + entityIdToListMap.clear(); + for (IdMsgPair pair : orderedMsgList) { + EntityId entityId = getEntityId(pair.msg.getValue()); + if (entityId != null) { + msgToEntityIdMap.put(pair.uuid, entityId); + entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); + } + } + } + + protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..cd8a97e82c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@Slf4j +public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + try { + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } catch (InvalidProtocolBufferException e) { + log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); + return null; + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..b258c6db1b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ef45b983fc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final AtomicInteger msgIdx = new AtomicInteger(0); + private volatile BiConsumer> msgConsumer; + private volatile UUID expectedMsgId; + + public SequentialTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + msgIdx.set(0); + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + } + + @Override + protected void doOnSuccess(UUID id) { + if (expectedMsgId.equals(id)) { + msgIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int idx = msgIdx.get(); + if (idx < listSize) { + IdMsgPair pair = orderedMsgList.get(idx); + expectedMsgId = pair.uuid; + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + } + msgConsumer.accept(pair.uuid, pair.msg); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index e67936e294..8e0fcaa74a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.ProcessingAttemptContext; import java.util.UUID; import java.util.concurrent.ConcurrentMap; @@ -31,24 +32,27 @@ public class TbRuleEngineProcessingResult { @Getter private final boolean timeout; @Getter - private final ConcurrentMap> pendingMap; - @Getter - private final ConcurrentMap> successMap; - @Getter - private final ConcurrentMap> failureMap; - @Getter - private final ConcurrentMap exceptionsMap; + private final ProcessingAttemptContext ctx; - public TbRuleEngineProcessingResult(boolean timeout, - ConcurrentMap> pendingMap, - ConcurrentMap> successMap, - ConcurrentMap> failureMap, - ConcurrentMap exceptionsMap) { + public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { this.timeout = timeout; - this.pendingMap = pendingMap; - this.successMap = successMap; - this.failureMap = failureMap; - this.exceptionsMap = exceptionsMap; - this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); + this.ctx = ctx; + this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); + } + + public ConcurrentMap> getPendingMap() { + return ctx.getPendingMap(); + } + + public ConcurrentMap> getSuccessMap() { + return ctx.getSuccessMap(); + } + + public ConcurrentMap> getFailedMap() { + return ctx.getFailedMap(); + } + + public ConcurrentMap getExceptionsMap() { + return ctx.getExceptionsMap(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 1096bfbf7e..b6579b8dcb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -16,12 +16,10 @@ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -77,10 +75,10 @@ public class TbRuleEngineProcessingStrategyFactory { return new TbRuleEngineProcessingDecision(true, null); } else { if (retryCount == 0) { - initialTotalCount = result.getPendingMap().size() + result.getFailureMap().size() + result.getSuccessMap().size(); + initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); } retryCount++; - double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); + double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); @@ -90,7 +88,7 @@ public class TbRuleEngineProcessingStrategyFactory { } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); if (retryFailed) { - result.getFailureMap().forEach(toReprocess::put); + result.getFailedMap().forEach(toReprocess::put); } if (retryTimeout) { result.getPendingMap().forEach(toReprocess::put); @@ -125,7 +123,7 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailureMap().size(), result.getPendingMap().size()); + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..7b22da97db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +public interface TbRuleEngineSubmitStrategy { + + void init(List> msgs); + + ConcurrentMap> getPendingMap(); + + void submitAttempt(BiConsumer> msgConsumer); + + void update(ConcurrentMap> reprocessMap); + + void onSuccess(UUID id); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java new file mode 100644 index 0000000000..f5a7457c17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; + +@Component +@Slf4j +public class TbRuleEngineSubmitStrategyFactory { + + public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "BURST": + return new BurstTbRuleEngineSubmitStrategy(name); + case "BATCH": + return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); + case "SEQUENTIAL_WITHIN_ORIGINATOR": + return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL_WITHIN_TENANT": + return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL": + return new SequentialTbRuleEngineSubmitStrategy(name); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index ce5065a64d..6d70056af0 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -123,7 +123,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); UUID requestId = request.getId(); localToDeviceRpcRequests.put(requestId, rpcMsg); - actorContext.getAppActor().tell(rpcMsg, ActorRef.noSender()); + actorContext.tell(rpcMsg, ActorRef.noSender()); scheduleToDeviceTimeout(request, requestId); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f08d2baf46..4c4955fea6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -578,27 +578,35 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - name: "Main" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java index c1a8fd883d..0d21c59c9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -20,13 +20,9 @@ import lombok.Data; @Data public class TbRuleEngineQueueAckStrategyConfiguration { -// @Value("${type}") private String type; -// @Value("${retries:3}") private int retries; -// @Value("${failure_percentage:0}") private double failurePercentage; -// @Value("${pause_between_retries:3}") private long pauseBetweenRetries; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index f89a615d7d..c5a24f20c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -25,6 +25,7 @@ public class TbRuleEngineQueueConfiguration { private int pollInterval; private int partitions; private String packProcessingTimeout; - private TbRuleEngineQueueAckStrategyConfiguration ackStrategy; + private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; + private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java new file mode 100644 index 0000000000..e39dc0c322 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; + +@Data +public class TbRuleEngineQueueSubmitStrategyConfiguration { + + private String type; + private int batchSize; + +} From 7b3d47526792fd5a26fdaa9233afcc69f110ed5b Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Thu, 9 Apr 2020 18:41:37 +0300 Subject: [PATCH 037/102] [2.5] created rabbitmq queue (#2589) * created main classes for RabbitMq queue * created temp version rabbitmq * Merge branch 'develop/2.5' of https://github.com/thingsboard/thingsboard into develop/2.5-rabbitmq # Conflicts: # common/queue/pom.xml * rabbit improvements --- .../DefaultTbRuleEngineConsumerService.java | 2 + .../src/main/resources/thingsboard.yml | 10 + common/queue/pom.xml | 4 + .../queue/common/DefaultTbQueueMsg.java | 9 +- .../common/DefaultTbQueueRequestTemplate.java | 7 +- .../DefaultTbQueueResponseTemplate.java | 5 - .../RabbitMqMonolithQueueFactory.java | 128 +++++++++++++ .../provider/RabbitMqTbCoreQueueFactory.java | 117 ++++++++++++ .../RabbitMqTbRuleEngineQueueFactory.java | 100 ++++++++++ .../RabbitMqTransportQueueFactory.java | 97 ++++++++++ .../queue/rabbitmq/TbRabbitMqAdmin.java | 82 +++++++++ .../rabbitmq/TbRabbitMqConsumerTemplate.java | 173 ++++++++++++++++++ .../rabbitmq/TbRabbitMqProducerTemplate.java | 113 ++++++++++++ .../queue/rabbitmq/TbRabbitMqSettings.java | 65 +++++++ .../queue/sqs/TbAwsSqsProducerTemplate.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 10 + 16 files changed, 904 insertions(+), 20 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 38cc7b8dbd..c8f126b26c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -119,6 +119,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (submitExecutor != null) { submitExecutor.shutdownNow(); } + + ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); } @Override diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4c4955fea6..affd7e7908 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -550,6 +550,16 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/pom.xml b/common/queue/pom.xml index f55f69ece1..668da07e29 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -64,6 +64,10 @@ com.microsoft.azure azure-servicebus + + com.rabbitmq + amqp-client + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index c7e439ef7d..7584e8c2d0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -17,7 +17,6 @@ package org.thingsboard.server.queue.common; import lombok.Data; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; @@ -25,13 +24,7 @@ import java.util.UUID; public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; - private DefaultTbQueueMsgHeaders headers; - - - public DefaultTbQueueMsg(UUID key, byte[] data) { - this.key = key; - this.data = data; - } + private final DefaultTbQueueMsgHeaders headers; public DefaultTbQueueMsg(TbQueueMsg msg) { this.key = msg.getKey(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 6ae9b9a33c..e8ebb934d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -20,7 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueConsumer; @@ -28,7 +28,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -128,10 +127,6 @@ public class DefaultTbQueueRequestTemplate> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java new file mode 100644 index 0000000000..3bf6d4667c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") +public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..52d28cccd7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") +public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueAdmin admin; + + public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqSettings rabbitMqSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java new file mode 100644 index 0000000000..dc6154255c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbRabbitMqProducerTemplate> producerTemplate = + new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + + TbRabbitMqConsumerTemplate> consumerTemplate = + new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java new file mode 100644 index 0000000000..fbd678045b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqAdmin implements TbQueueAdmin { + + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { + this.rabbitMqSettings = rabbitMqSettings; + + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + try { + channel.queueDeclare(topic, false, false, false, null); + } catch (IOException e) { + log.error("Failed to bind queue: [{}]", topic, e); + } + } + + @PreDestroy + private void destroy() { + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close Chanel.", e); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close Connection.", e); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java new file mode 100644 index 0000000000..25d7719163 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.GetResponse; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@Slf4j +public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + private volatile Set partitions; + private volatile boolean subscribed; + private volatile Set queues; + private volatile boolean stopped; + + public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.rabbitMqSettings = rabbitMqSettings; + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + + queues.forEach(admin::createTopicIfNotExists); + subscribed = true; + } + + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode message: [{}].", message); + throw new RuntimeException("Failed to decode message.", e); + } + }).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } + } + + public T decode(GetResponse message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java new file mode 100644 index 0000000000..91b46213a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbRabbitMqSettings rabbitMqSettings; + private ListeningExecutorService producerExecutor; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.rabbitMqSettings = rabbitMqSettings; + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + AMQP.BasicProperties properties = new AMQP.BasicProperties(); + try { + channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); + if (callback != null) { + callback.onSuccess(null); + } + } catch (IOException e) { + log.error("Failed publish message: [{}].", msg, e); + if (callback != null) { + callback.onFailure(e); + } + } + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java new file mode 100644 index 0000000000..e0156e6dc8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.ConnectionFactory; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +@Component +@Data +public class TbRabbitMqSettings { + @Value("${queue.rabbitmq.exchange_name:}") + private String exchangeName; + @Value("${queue.rabbitmq.host:}") + private String host; + @Value("${queue.rabbitmq.port:}") + private int port; + @Value("${queue.rabbitmq.virtual_host:}") + private String virtualHost; + @Value("${queue.rabbitmq.username:}") + private String username; + @Value("${queue.rabbitmq.password:}") + private String password; + @Value("${queue.rabbitmq.automatic_recovery_enabled:}") + private boolean automaticRecoveryEnabled; + @Value("${queue.rabbitmq.connection_timeout:}") + private int connectionTimeout; + @Value("${queue.rabbitmq.handshake_timeout:}") + private int handshakeTimeout; + + private ConnectionFactory connectionFactory; + + @PostConstruct + private void init() { + connectionFactory = new ConnectionFactory(); + connectionFactory.setHost(host); + connectionFactory.setPort(port); + connectionFactory.setVirtualHost(virtualHost); + connectionFactory.setUsername(username); + connectionFactory.setPassword(password); + connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); + connectionFactory.setConnectionTimeout(connectionTimeout); + connectionFactory.setHandshakeTimeout(handshakeTimeout); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 6eed92d296..2d85539184 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -78,7 +78,7 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { SendMessageRequest sendMsgRequest = new SendMessageRequest(); sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); sendMsgRequest.withMessageGroupId(msg.getKey().toString()); ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 9b04867cfe..395f1d09e0 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -88,6 +88,16 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 1dd3334825212603da640af693f1f95e274943c5 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 9 Apr 2020 14:44:35 +0300 Subject: [PATCH 038/102] moved jsinvoke.proto to queue, fixed js-executor, added createRemoteJsRequestTemplate to RuleEngine and Core factories --- .../service/script/RemoteJsInvokeService.java | 67 +++++-------------- .../queue/common/TbProtoJsQueueMsg.java | 43 ++++++++++++ .../server/queue/common/TbProtoQueueMsg.java | 2 +- .../provider/AwsSqsMonolithQueueFactory.java | 8 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 8 +++ .../AwsSqsTbRuleEngineQueueFactory.java | 8 +++ .../InMemoryMonolithQueueFactory.java | 8 +++ .../provider/KafkaMonolithQueueFactory.java | 48 ++++++++++++- .../provider/KafkaTbCoreQueueFactory.java | 48 ++++++++++++- .../KafkaTbRuleEngineQueueFactory.java | 48 ++++++++++++- .../provider/PubSubMonolithQueueFactory.java | 8 +++ .../provider/PubSubTbCoreQueueFactory.java | 8 +++ .../PubSubTbRuleEngineQueueFactory.java | 12 +++- .../ServiceBusMonolithQueueFactory.java | 8 +++ ...java => ServiceBusTbCoreQueueFactory.java} | 24 ++++--- .../ServiceBusTbRuleEngineQueueFactory.java | 8 +++ .../queue/provider/TbCoreQueueFactory.java | 14 ++-- .../provider/TbRuleEngineQueueFactory.java | 12 ++-- .../TbQueueRemoteJsInvokeSettings.java | 42 ++++++++++++ .../queue}/src/main/proto/jsinvoke.proto | 0 .../api/jsInvokeMessageProcessor.js | 5 +- 21 files changed, 353 insertions(+), 76 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{ServiceBusTbCoreQueueProvider.java => ServiceBusTbCoreQueueFactory.java} (84%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java rename {application => common/queue}/src/main/proto/jsinvoke.proto (100%) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index b0554decea..8d1f9d662a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -21,14 +21,15 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -41,28 +42,13 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) +@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.request_topic}") - private String requestTopic; - - @Value("${js.remote.response_topic_prefix}") - private String responseTopicPrefix; - - @Value("${js.remote.max_pending_requests}") - private long maxPendingRequests; - @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; - @Value("${js.remote.response_poll_interval}") - private int responsePollDuration; - - @Value("${js.remote.response_auto_commit_interval}") - private int autoCommitInterval; - @Getter @Value("${js.remote.max_errors}") private int maxErrors; @@ -94,43 +80,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private DefaultTbQueueRequestTemplate, TbProtoQueueMsg> defaultTemplate; + @Autowired + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); -// requestBuilder.settings(kafkaSettings); -// requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); -// requestBuilder.defaultTopic(requestTopic); -// requestBuilder.encoder(new RemoteJsRequestEncoder()); - TbQueueProducer> producer; - -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); -// responseBuilder.settings(kafkaSettings); -// responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); -// responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); -// responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new RemoteJsResponseDecoder()); -// responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); -// -// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder -// builder = TbKafkaRequestTemplate.builder(); -// builder.requestTemplate(requestBuilder.build()); -// builder.responseTemplate(responseBuilder.build()); -// builder.maxPendingRequests(maxPendingRequests); -// builder.maxRequestTimeout(maxRequestsTimeout); -// builder.pollInterval(responsePollDuration); -// defaultTemplate = builder.build(); -// defaultTemplate.init(); + requestTemplate.init(); } @PreDestroy public void destroy() { - if (defaultTemplate != null) { - defaultTemplate.stop(); + if (requestTemplate != null) { + requestTemplate.stop(); } } @@ -147,7 +110,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @@ -199,7 +162,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @Override @@ -239,7 +202,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java new file mode 100644 index 0000000000..07417c4a9e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class TbProtoJsQueueMsg extends TbProtoQueueMsg { + + public TbProtoJsQueueMsg(UUID key, T value) { + super(key, value); + } + + public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + super(key, value, headers); + } + + @Override + public byte[] getData() { + try { + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 8823002530..2eb76950dc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -25,7 +25,7 @@ import java.util.UUID; public class TbProtoQueueMsg implements TbQueueMsg { private final UUID key; - private final T value; + protected final T value; private final TbQueueMsgHeaders headers; public TbProtoQueueMsg(UUID key, T value) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 9f70345301..26da11aa3b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -18,10 +18,13 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -125,4 +128,9 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 770e7fa65c..87aba51843 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -114,4 +117,9 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index f87f108453..d83b760c19 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,6 +26,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -96,4 +99,9 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a23c93e015..baad96182f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,6 +28,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -110,4 +113,9 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index fa36f8284b..dae4221e91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,18 +30,25 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -50,13 +60,15 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,6 +76,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -175,4 +188,37 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 5e46d67eed..e148416bb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,16 +30,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @@ -47,18 +57,21 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings) { + TbQueueTransportApiSettings transportApiSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -148,4 +161,37 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index eef10f35dc..8262d27354 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,16 +28,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -44,16 +54,19 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings) { + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -124,4 +137,37 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index f00bd7caef..b1afc61dbd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -133,4 +136,9 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index da2fe3acdf..edfcdc3d8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,6 +28,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -110,4 +113,9 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index da65d583c1..101f335536 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,15 +26,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @@ -99,4 +102,9 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 914e200fc9..8b01c38607 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,9 +29,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -131,4 +134,9 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul public TbQueueProducer> createTransportApiResponseProducer() { return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 334af58133..523ac88bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,9 +28,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +42,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") -public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { +public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbServiceBusSettings serviceBusSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -49,13 +52,13 @@ public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueAdmin admin; - public ServiceBusTbCoreQueueProvider(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -113,4 +116,9 @@ public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 3ceb837606..8a2f9b396b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,9 +26,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -96,4 +99,9 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 1a3ba1ab98..59a8f449b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos.*; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. @@ -94,5 +98,5 @@ public interface TbCoreQueueFactory { */ TbQueueProducer> createTransportApiResponseProducer(); - + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 300fb986a9..561b3e84ea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; /** @@ -82,4 +85,5 @@ public interface TbRuleEngineQueueFactory { */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java new file mode 100644 index 0000000000..cdad407b19 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRemoteJsInvokeSettings { + @Value("${js.remote.request_topic}") + private String requestTopic; + + @Value("${js.remote.response_topic_prefix}") + private String responseTopic; + + @Value("${js.remote.max_pending_requests}") + private long maxPendingRequests; + + @Value("${js.remote.response_poll_interval}") + private int responsePollInterval; + + @Value("${js.remote.response_auto_commit_interval}") + private int autoCommitInterval; + + @Value("${js.remote.max_requests_timeout}") + private long maxRequestsTimeout; +} diff --git a/application/src/main/proto/jsinvoke.proto b/common/queue/src/main/proto/jsinvoke.proto similarity index 100% rename from application/src/main/proto/jsinvoke.proto rename to common/queue/src/main/proto/jsinvoke.proto diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index c17b1ddde4..f0facf8cc1 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,6 +19,7 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; +let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -43,6 +44,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); + headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -148,7 +150,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse + value: rawResponse, + headers: headers } ] } From 8442159811021264cea5c535aec3a57661bb9b52 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 13 Apr 2020 09:34:35 +0300 Subject: [PATCH 039/102] [2.5] vulnerabilities (#2608) * Version updates * added @Retention(RetentionPolicy.RUNTIME) for all custom Dao annotations. * changed getId, setId to getUuid, setUuid from BaseEntity, improvement AbstractControllerTest * fix rabbitmq * fix RuleChainManagerActor * refactored InMemory queue Co-authored-by: Andrii Shvaika --- .../ruleChain/RuleChainManagerActor.java | 4 +- .../actors/service/ContextAwareActor.java | 4 +- .../controller/AbstractControllerTest.java | 4 +- .../thingsboard/server/dao/util/HsqlDao.java | 4 ++ .../server/dao/util/NoSqlAnyDao.java | 4 ++ .../thingsboard/server/dao/util/NoSqlDao.java | 4 ++ .../server/dao/util/NoSqlTsDao.java | 4 ++ .../thingsboard/server/dao/util/PsqlDao.java | 4 ++ .../server/dao/util/PsqlTsAnyDao.java | 4 ++ .../thingsboard/server/dao/util/SqlDao.java | 4 ++ .../server/dao/util/SqlTsAnyDao.java | 4 ++ .../thingsboard/server/dao/util/SqlTsDao.java | 4 ++ .../server/dao/util/TimescaleDBTsDao.java | 4 ++ .../server/queue/memory/InMemoryStorage.java | 20 +++--- .../queue/memory/InMemoryTbQueueConsumer.java | 38 +++++++++-- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../InMemoryMonolithQueueFactory.java | 66 ++++++++++--------- .../InMemoryTbTransportQueueFactory.java | 42 ++++++------ .../RabbitMqMonolithQueueFactory.java | 8 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 8 +++ .../RabbitMqTbRuleEngineQueueFactory.java | 8 +++ .../CassandraBaseComponentDescriptorDao.java | 6 +- .../dao/event/CassandraBaseEventDao.java | 6 +- .../server/dao/model/BaseEntity.java | 4 +- .../server/dao/model/BaseSqlEntity.java | 5 +- .../dao/model/nosql/AdminSettingsEntity.java | 4 +- .../server/dao/model/nosql/AlarmEntity.java | 4 +- .../server/dao/model/nosql/AssetEntity.java | 4 +- .../dao/model/nosql/AuditLogEntity.java | 4 +- .../nosql/ComponentDescriptorEntity.java | 4 +- .../dao/model/nosql/CustomerEntity.java | 4 +- .../dao/model/nosql/DashboardEntity.java | 4 +- .../dao/model/nosql/DashboardInfoEntity.java | 4 +- .../model/nosql/DeviceCredentialsEntity.java | 4 +- .../server/dao/model/nosql/DeviceEntity.java | 4 +- .../dao/model/nosql/EntityViewEntity.java | 10 +++ .../server/dao/model/nosql/EventEntity.java | 4 +- .../dao/model/nosql/RuleChainEntity.java | 4 +- .../dao/model/nosql/RuleNodeEntity.java | 4 +- .../server/dao/model/nosql/TenantEntity.java | 4 +- .../model/nosql/UserCredentialsEntity.java | 4 +- .../server/dao/model/nosql/UserEntity.java | 4 +- .../dao/model/nosql/WidgetTypeEntity.java | 4 +- .../dao/model/nosql/WidgetsBundleEntity.java | 4 +- .../dao/model/sql/AdminSettingsEntity.java | 2 +- .../server/dao/model/sql/AlarmEntity.java | 2 +- .../server/dao/model/sql/AssetEntity.java | 2 +- .../server/dao/model/sql/AuditLogEntity.java | 6 +- .../model/sql/ComponentDescriptorEntity.java | 5 +- .../server/dao/model/sql/CustomerEntity.java | 6 +- .../server/dao/model/sql/DashboardEntity.java | 6 +- .../dao/model/sql/DashboardInfoEntity.java | 6 +- .../model/sql/DeviceCredentialsEntity.java | 6 +- .../server/dao/model/sql/DeviceEntity.java | 6 +- .../dao/model/sql/EntityViewEntity.java | 6 +- .../server/dao/model/sql/EventEntity.java | 6 +- .../server/dao/model/sql/RuleChainEntity.java | 6 +- .../server/dao/model/sql/RuleNodeEntity.java | 6 +- .../server/dao/model/sql/TenantEntity.java | 6 +- .../dao/model/sql/UserCredentialsEntity.java | 6 +- .../server/dao/model/sql/UserEntity.java | 6 +- .../dao/model/sql/WidgetTypeEntity.java | 6 +- .../dao/model/sql/WidgetsBundleEntity.java | 2 +- .../dao/nosql/CassandraAbstractModelDao.java | 6 +- .../server/dao/sql/JpaAbstractDao.java | 4 +- .../server/dao/sql/asset/JpaAssetDao.java | 8 +-- .../server/dao/sql/audit/JpaAuditLogDao.java | 9 +-- ...ctComponentDescriptorInsertRepository.java | 8 +-- ...qlComponentDescriptorInsertRepository.java | 2 +- .../JpaBaseComponentDescriptorDao.java | 4 +- .../dao/sql/customer/JpaCustomerDao.java | 2 +- .../sql/dashboard/JpaDashboardInfoDao.java | 2 +- .../server/dao/sql/device/JpaDeviceDao.java | 10 +-- .../dao/sql/entityview/JpaEntityViewDao.java | 8 +-- .../event/AbstractEventInsertRepository.java | 2 +- .../sql/event/HsqlEventInsertRepository.java | 2 +- .../server/dao/sql/event/JpaBaseEventDao.java | 13 ++-- .../dao/sql/relation/JpaRelationDao.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 2 +- .../server/dao/sql/tenant/JpaTenantDao.java | 2 +- .../server/dao/sql/user/JpaUserDao.java | 4 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 6 +- ...stractChunkedAggregationTimeseriesDao.java | 4 +- .../timescale/TimescaleTimeseriesDao.java | 4 +- pom.xml | 60 +++++++++++++---- 85 files changed, 369 insertions(+), 242 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index b1517b9406..a90d371f2a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -72,11 +72,11 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } } - public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId) { + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); } - public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId, Function provider) { + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { return actors.computeIfAbsent(ruleChainId, eId -> { RuleChain ruleChain = provider.apply(eId); return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java index 84818bd3e4..a2c0d08ade 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java @@ -16,14 +16,14 @@ package org.thingsboard.server.actors.service; import akka.actor.Terminated; -import akka.actor.UntypedActor; +import akka.actor.UntypedAbstractActor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.TbActorMsg; -public abstract class ContextAwareActor extends UntypedActor { +public abstract class ContextAwareActor extends UntypedAbstractActor { protected final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 50e3580405..3e9f8c7853 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -114,9 +114,7 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), - MediaType.APPLICATION_JSON.getSubtype(), - Charset.forName("utf8")); + protected MediaType contentType = MediaType.APPLICATION_JSON; protected MockMvc mockMvc; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index 96a36e1f65..a41dd2ee21 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index 20954b8ac2..bcf1222988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java index 8b642de026..dde0a9a176 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "cassandra") public @interface NoSqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index ebd0ef7d28..7b10e0cf03 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index ef73ec8f90..a888926bd0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index f2a8800032..a215a16b29 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 0eea3367e1..408f81a845 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql") public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9a43c530a2..9be3321988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index abba0e985b..0b665c5695 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 20745d790c..0541a47879 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index a61bf560ee..c83fa02fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -21,7 +21,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -30,10 +29,12 @@ import java.util.concurrent.TimeUnit; @Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; - private final Map> storage; + private final ConcurrentHashMap> storage; + private volatile boolean stopped; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); + stopped = false; } public static InMemoryStorage getInstance() { @@ -67,19 +68,20 @@ public final class InMemoryStorage { entities.add((T) other); } } + if (entities.size() > 0) { + storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).addAll(entities); + } return entities; } catch (InterruptedException e) { - log.warn("Queue was interrupted", e); - return Collections.emptyList(); + if (!stopped) { + log.warn("Queue was interrupted", e); + } } } return Collections.emptyList(); } - public void commit(String topic) { - //TODO: 2.5 Until someone calls commit you should not allow to poll new elements. - if (storage.containsKey(topic)) { -// storage.get(topic).remove(); - } + public void stop() { + stopped = true; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 8f226ff2e9..2b81839540 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -15,18 +15,26 @@ */ package org.thingsboard.server.queue.memory; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +@Slf4j public class InMemoryTbQueueConsumer implements TbQueueConsumer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); + private volatile Set partitions; + private volatile boolean stopped; + private volatile boolean subscribed; public InMemoryTbQueueConsumer(String topic) { this.topic = topic; + stopped = false; } private final String topic; @@ -38,26 +46,44 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void subscribe() { - + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = true; } @Override public void subscribe(Set partitions) { - + this.partitions = partitions; + subscribed = true; } @Override public void unsubscribe() { - + stopped = true; } @Override public List poll(long durationInMillis) { - return storage.get(topic, durationInMillis); + if (subscribed) { + List messages = partitions + .stream() + .map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) + .flatMap(List::stream) + .map(msg -> (T) msg).collect(Collectors.toList()); + if (messages.size() > 0) { + return messages; + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to sleep.", e); + } + } + } + return Collections.emptyList(); } @Override public void commit() { - storage.commit(topic); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index cb1ff939d5..cfcd788a16 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -39,7 +39,7 @@ public class InMemoryTbQueueProducer implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - boolean result = storage.put(tpi.getTopic(), msg); + boolean result = storage.put(tpi.getFullTopicName(), msg); if (result) { if (callback != null) { callback.onSuccess(null); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index baad96182f..fbdbeaab0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -18,19 +18,16 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -44,74 +41,79 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings notificationSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public InMemoryMonolithQueueFactory(TbQueueCoreSettings coreSettings, + public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; - this.notificationSettings = notificationSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createToCoreMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); + public TbQueueProducer> createTransportApiResponseProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index c779127e1f..29660f2461 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; @@ -29,10 +28,9 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -40,32 +38,32 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { - - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings notificationSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceInfoProvider serviceInfoProvider; - public InMemoryTbTransportQueueFactory(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; + public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceInfoProvider serviceInfoProvider) { this.transportApiSettings = transportApiSettings; - this.notificationSettings = notificationSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceInfoProvider = serviceInfoProvider; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + InMemoryTbQueueProducer> producerTemplate = + new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumerTemplate = + new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(topic -> Futures.immediateFuture(null)); - templateBuilder.requestTemplate(producer); - templateBuilder.responseTemplate(consumer); + templateBuilder.queueAdmin(topic -> { + }); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); @@ -74,16 +72,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); + return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index e0ff9e1032..ff4a69e2e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -18,10 +18,13 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -125,4 +128,9 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 3bf6d4667c..5708e0738c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -114,4 +117,9 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 52d28cccd7..e2755938d9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -26,6 +27,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -97,4 +100,9 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java index 664d7b382f..68e398131b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java @@ -151,12 +151,12 @@ public class CassandraBaseComponentDescriptorDao extends CassandraAbstractSearch } private Optional saveIfNotExist(TenantId tenantId, ComponentDescriptorEntity entity) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } ResultSet rs = executeRead(tenantId, QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.COMPONENT_DESCRIPTOR_NAME_PROPERTY, entity.getName()) .value(ModelConstants.COMPONENT_DESCRIPTOR_CLASS_PROPERTY, entity.getClazz()) .value(ModelConstants.COMPONENT_DESCRIPTOR_TYPE_PROPERTY, entity.getType()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java index bdd0201aa5..f2660e512d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java @@ -184,11 +184,11 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao> saveAsync(TenantId tenantId, EventEntity entity, boolean ifNotExists, int ttl) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } Insert insert = QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.EVENT_TENANT_ID_PROPERTY, entity.getTenantId()) .value(ModelConstants.EVENT_ENTITY_TYPE_PROPERTY, entity.getEntityType()) .value(ModelConstants.EVENT_ENTITY_ID_PROPERTY, entity.getEntityId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index 386220df55..b5bda06a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getId(); + UUID getUuid(); - void setId(UUID id); + void setUuid(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index d75262e311..44fd70f977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,14 +35,15 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getId() { + public UUID getUuid() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - public void setId(UUID id) { + @Override + public void setUuid(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java index 93cc934a06..5d8f5d671f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java @@ -61,11 +61,11 @@ public final class AdminSettingsEntity implements BaseEntity { this.jsonValue = adminSettings.getJsonValue(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java index 0cc8dfab72..d2e906121f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java @@ -141,11 +141,11 @@ public final class AlarmEntity implements BaseEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java index 2554e0c3ec..57ec0ee511 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java @@ -94,11 +94,11 @@ public final class AssetEntity implements SearchTextEntity { this.additionalInfo = asset.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java index 04e77aad8a..b406a6b341 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java @@ -94,12 +94,12 @@ public class AuditLogEntity implements BaseEntity { private String actionFailureDetails; @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java index 9a6827aa05..1af3b8a8fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java @@ -98,12 +98,12 @@ public class ComponentDescriptorEntity implements SearchTextEntity { this.additionalInfo = customer.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java index 9fe999a0e5..a48305f630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java @@ -98,11 +98,11 @@ public final class DashboardEntity implements SearchTextEntity { this.configuration = dashboard.getConfiguration(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java index b54252fdfc..69eb43744c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java @@ -91,11 +91,11 @@ public class DashboardInfoEntity implements SearchTextEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java index 6900014d21..316386f7c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java @@ -74,11 +74,11 @@ public final class DeviceCredentialsEntity implements BaseEntity { this.additionalInfo = device.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java index 80bc9a3276..5cf13665b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java @@ -161,4 +161,14 @@ public class EntityViewEntity implements SearchTextEntity { entityView.setAdditionalInfo(additionalInfo); return entityView; } + + @Override + public UUID getUuid() { + return getId(); + } + + @Override + public void setUuid(UUID id) { + this.id = id; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java index 2645d29df3..8c753a743a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java @@ -94,12 +94,12 @@ public class EventEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java index bb6a194801..7021ff708e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java @@ -102,12 +102,12 @@ public class RuleChainEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java index e2dc2fa244..9a086fe568 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java @@ -96,12 +96,12 @@ public class RuleNodeEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java index 0ce7990755..29a51e2539 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java @@ -110,11 +110,11 @@ public final class TenantEntity implements SearchTextEntity { this.additionalInfo = tenant.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java index 50b54bd8c9..6854d61f39 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java @@ -75,11 +75,11 @@ public final class UserCredentialsEntity implements BaseEntity this.resetToken = userCredentials.getResetToken(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java index d8399fc2b5..0c7625da78 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java @@ -101,11 +101,11 @@ public final class UserEntity implements SearchTextEntity { this.additionalInfo = user.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java index 65bc19f421..24ac3980cf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java @@ -82,12 +82,12 @@ public final class WidgetTypeEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java index b70a0f095b..8c11c05685 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java @@ -82,12 +82,12 @@ public final class WidgetsBundleEntity implements SearchTextEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setId(adminSettings.getId().getId()); + this.setUuid(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index 14adcdcd9c..fb4efd25d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -114,7 +114,7 @@ public final class AlarmEntity extends BaseSqlEntity implements BaseEntit public AlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setId(alarm.getId().getId()); + this.setUuid(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java index 0471d4a97d..6eff4cd521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java @@ -78,7 +78,7 @@ public final class AssetEntity extends BaseSqlEntity implements SearchTex public AssetEntity(Asset asset) { if (asset.getId() != null) { - this.setId(asset.getId().getId()); + this.setUuid(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index 4de738ac73..e2573ea0d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setId(auditLog.getId().getId()); + this.setUuid(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(getId())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); + AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index 26946cf207..d5b287fe9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,7 +34,6 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; -import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -72,7 +71,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setId(customer.getId().getId()); + this.setUuid(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(getId())); - customer.setCreatedTime(UUIDs.unixTimestamp(getId())); + Customer customer = new Customer(new CustomerId(this.getUuid())); + customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index b606c35d4e..07cec4d12f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setId(dashboard.getId().getId()); + this.setUuid(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index b1ecb2572b..f26cf21934 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setId(dashboardInfo.getId().getId()); + this.setUuid(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index 1d47c7d63c..ba9d7ce637 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements SearchT public DeviceEntity(Device device) { if (device.getId() != null) { - this.setId(device.getId().getId()); + this.setUuid(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -95,8 +95,8 @@ public final class DeviceEntity extends BaseSqlEntity implements SearchT @Override public Device toData() { - Device device = new Device(new DeviceId(getId())); - device.setCreatedTime(UUIDs.unixTimestamp(getId())); + Device device = new Device(new DeviceId(this.getUuid())); + device.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java index 6a755d2803..fd446cc6ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java @@ -99,7 +99,7 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc public EntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setId(entityView.getId().getId()); + this.setUuid(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -136,8 +136,8 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc @Override public EntityView toData() { - EntityView entityView = new EntityView(new EntityViewId(getId())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); + EntityView entityView = new EntityView(new EntityViewId(this.getUuid())); + entityView.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..6cade0f576 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -75,7 +75,7 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setId(ruleChain.getUuidId()); + this.setUuid(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index f36edc260b..1a7866418a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setId(ruleNode.getUuidId()); + this.setUuid(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 239d3698b4..10b00f38ea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -82,7 +82,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setId(tenant.getId().getId()); + this.setUuid(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -113,8 +113,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(getId())); - tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); + Tenant tenant = new Tenant(new TenantId(this.getUuid())); + tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 70593ca610..6957f87961 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setId(userCredentials.getId().getId()); + this.setUuid(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 0100110daa..3af671583e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setId(user.getId().getId()); + this.setUuid(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(getId())); - user.setCreatedTime(UUIDs.unixTimestamp(getId())); + User user = new User(new UserId(this.getUuid())); + user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 8e81b30e8f..0cb69b9d8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setId(widgetType.getId().getId()); + this.setUuid(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 7df6ea9fe7..75ea383090 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setId(widgetsBundle.getId().getId()); + this.setUuid(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java index dd56136e76..2da9915bb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java @@ -132,10 +132,10 @@ public abstract class CassandraAbstractModelDao, D> exte protected EntityResultSet saveWithResult(TenantId tenantId, E entity) { log.debug("Save entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } else if (isDeleteOnSave()) { - removeById(tenantId, entity.getId()); + removeById(tenantId, entity.getUuid()); } Statement saveStatement = getSaveQuery(entity); saveStatement.setConsistencyLevel(cluster.getDefaultWriteConsistencyLevel()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index d3ce07f901..edc9c97a9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index aaddc15eb8..c4482a3247 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -69,7 +69,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -86,7 +86,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -109,7 +109,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -121,7 +121,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index c0a260e9da..566d9c3225 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -16,8 +16,6 @@ package org.thingsboard.server.dao.sql.audit; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -39,14 +37,11 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTimeDao; import org.thingsboard.server.dao.util.SqlDao; -import javax.annotation.PreDestroy; import javax.persistence.criteria.Predicate; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.concurrent.Executors; -import static org.springframework.data.jpa.domain.Specifications.where; import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @@ -118,8 +113,8 @@ public class JpaAuditLogDao extends JpaAbstractDao imp Specification timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, customerId, userId, actionTypes); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(auditLogRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(auditLogRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } private Specification getEntityFieldsSpec(UUID tenantId, EntityId entityId, CustomerId customerId, UserId userId, List actionTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index c634e5a064..d8272e3a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index e281bdeec4..c33dd4f5e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index 938b030d53..e66937166f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -94,7 +94,7 @@ public class JpaBaseComponentDescriptorDao extends JpaAbstractSearchTextDao deviceRepository.findByTenantId( fromTimeUUID(tenantId), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } else { return DaoUtil.convertDataList( deviceRepository.findByTenantId( fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } @@ -95,7 +95,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -118,7 +118,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -130,7 +130,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 91a9181908..385a901dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -70,7 +70,7 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, eventType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(eventRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(eventRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } @Override @@ -130,7 +129,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "toId"); Specification fieldsSpec = getEntityFieldsSpec(from, relationType, typeGroup, childType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, "toId"); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, "toId"); return service.submit(() -> - DaoUtil.convertDataList(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); + DaoUtil.convertDataList(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 415cbd15c1..b1e4745223 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -60,7 +60,7 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao region, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : UUIDConverter.fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 33b7c3c1e9..f30f6ba6d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -71,7 +71,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.TENANT_ADMIN, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -84,7 +84,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 228c596642..af6785e69e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -68,7 +68,7 @@ public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index bf4cf5d9e6..0a2e210648 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( + PageRequest.of(0, query.getLimit(), + Sort.by(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/pom.xml b/pom.xml index 383fbf0edc..8876e68538 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.1.3.RELEASE - 5.1.5.RELEASE - 5.1.4.RELEASE - 2.1.5.RELEASE - 2.9.0 + 2.2.4.RELEASE + 5.2.2.RELEASE + 5.2.2.RELEASE + 2.2.4.RELEASE + 3.1.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.9.9.3 - 2.9.9 - 2.9.9 + 2.10.2 + 2.10.2 + 2.10.2 2.2.6 - 2.11 - 2.4.2 + 2.13 + 2.6.3 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.37.Final + 4.1.45.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.56 + 1.64 2.0.1 2.5.0 2.5.3 @@ -92,12 +92,15 @@ 4.1.1 2.57 2.7.7 - 1.23 + 1.25 + 1.3.10 1.11.747 1.84.0 3.2.0 1.5.0 1.4.3 + 1.9.4 + 3.2.2 @@ -915,6 +918,37 @@ uap-java ${ua-parser.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + commons-collections + commons-collections + ${commons-collections.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + org.apache.struts + struts-core + ${struts.version} + + + org.apache.struts + struts-taglib + ${struts.version} + + + org.apache.struts + struts-tiles + ${struts.version} + + From 7869fb5ef48eb2892aa5c0818c85017b0d86d08a Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 13 Apr 2020 10:24:40 +0300 Subject: [PATCH 040/102] added to tenant fields isolatedTbCore and isolatedTbRuleEngine (#2611) Co-authored-by: Andrew Shvayka --- .../CassandraDatabaseUpgradeService.java | 14 ++++ .../install/SqlDatabaseUpgradeService.java | 1 + .../server/common/data/Tenant.java | 22 +++++++ .../server/dao/model/ModelConstants.java | 2 + .../server/dao/model/nosql/TenantEntity.java | 35 ++++++++-- .../server/dao/model/sql/TenantEntity.java | 10 +++ .../resources/cassandra/schema-entities.cql | 2 + .../resources/sql/schema-entities-hsql.sql | 4 +- .../main/resources/sql/schema-entities.sql | 4 +- ui/src/app/locale/locale.constant-en_US.json | 6 +- ui/src/app/tenant/tenant-fieldset.tpl.html | 64 ++++++++++++------- 11 files changed, 134 insertions(+), 30 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java index 721d43bf9f..573d152635 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -290,6 +290,20 @@ public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUp log.info("Attributes updated."); } catch (InvalidQueryException e) { } + + String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; + String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; + + try { + log.info("Updating tenant..."); + cluster.getSession().execute(updateTenantCoreTableStmt); + Thread.sleep(2500); + + cluster.getSession().execute(updateTenantRuleEngineTableStmt); + Thread.sleep(2500); + log.info("Tenant updated."); + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..bcbd77ef28 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); log.info("Schema updated."); } break; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index b1f6a967b6..766d405e25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,6 +29,8 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; + private boolean isolatedTbCore; + private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -72,6 +74,22 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchText() { return getTitle(); @@ -84,6 +102,10 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); + builder.append(", isolatedTbCore="); + builder.append(isolatedTbCore); + builder.append(", isolatedTbRuleEngine="); + builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..2db9251e49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -112,6 +112,8 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; + public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; + public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java index 29a51e2539..cae31c2a31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java @@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.model.type.JsonCodec; @@ -55,16 +56,16 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_TITLE_PROPERTY) private String title; - + @Column(name = SEARCH_TEXT_PROPERTY) private String searchText; @Column(name = TENANT_REGION_PROPERTY) private String region; - + @Column(name = COUNTRY_PROPERTY) private String country; - + @Column(name = STATE_PROPERTY) private String state; @@ -89,6 +90,12 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_ADDITIONAL_INFO_PROPERTY, codec = JsonCodec.class) private JsonNode additionalInfo; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + public TenantEntity() { super(); } @@ -108,6 +115,8 @@ public final class TenantEntity implements SearchTextEntity { this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } public UUID getUuid() { @@ -206,6 +215,22 @@ public final class TenantEntity implements SearchTextEntity { this.additionalInfo = additionalInfo; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchTextSource() { return getTitle(); @@ -215,7 +240,7 @@ public final class TenantEntity implements SearchTextEntity { public void setSearchText(String searchText) { this.searchText = searchText; } - + public String getSearchText() { return searchText; } @@ -235,6 +260,8 @@ public final class TenantEntity implements SearchTextEntity { tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 10b00f38ea..1da71224f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,6 +72,12 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -95,6 +101,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -126,6 +134,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/resources/cassandra/schema-entities.cql b/dao/src/main/resources/cassandra/schema-entities.cql index de2b088cef..6d23ca8122 100644 --- a/dao/src/main/resources/cassandra/schema-entities.cql +++ b/dao/src/main/resources/cassandra/schema-entities.cql @@ -110,6 +110,8 @@ CREATE TABLE IF NOT EXISTS thingsboard.tenant ( phone text, email text, additional_info text, + isolated_tb_core boolean, + isolated_tb_rule_engine boolean, PRIMARY KEY (id, region) ); diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 758aaafb10..f28f7f5ebd 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 55893fc124..36dac1b40e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index a0750164ca..7bb6e6e5e7 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1508,7 +1508,11 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required" + "tenant-required": "Tenant is required", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index aa7751d479..35b3dee289 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,32 +15,50 @@ limitations under the License. --> -{{ 'tenant.manage-tenant-admins' | translate }} -{{ 'tenant.delete' | translate }} +{{ + 'tenant.manage-tenant-admins' | translate }} + +{{ 'tenant.delete' + | translate }} +
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
- - - - - - -
+
+ + + +
+
tenant.title-required
+
+ + + + + + + + + {{'tenant.isolated-tb-core' | translate}}
+ {{'tenant.isolated-tb-core-details' | translate}} + + + + + {{'tenant.isolated-tb-rule-engine' | translate}}
+ {{'tenant.isolated-tb-rule-engine-details' | translate}} + + +
From 5b8ce2fb69f18486643c195cdbd241a5e5044ffd Mon Sep 17 00:00:00 2001 From: "hamza.slama" Date: Tue, 7 Apr 2020 09:26:06 +0100 Subject: [PATCH 041/102] Move AlarmId To Id Package --- .../org/thingsboard/server/controller/AlarmController.java | 2 +- .../org/thingsboard/server/controller/BaseController.java | 2 +- .../java/org/thingsboard/server/dao/alarm/AlarmService.java | 2 +- .../java/org/thingsboard/server/common/data/alarm/Alarm.java | 1 + .../thingsboard/server/common/data/{alarm => id}/AlarmId.java | 4 +--- .../thingsboard/server/common/data/id/EntityIdFactory.java | 1 - .../org/thingsboard/server/dao/alarm/BaseAlarmService.java | 2 +- .../org/thingsboard/server/dao/entity/BaseEntityService.java | 2 +- .../org/thingsboard/server/dao/model/nosql/AlarmEntity.java | 2 +- .../org/thingsboard/server/dao/model/sql/AlarmEntity.java | 2 +- .../org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java | 2 +- .../src/main/java/org/thingsboard/rest/client/RestClient.java | 3 +-- .../engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java | 2 +- .../rule/engine/util/EntitiesFieldsAsyncLoader.java | 2 +- .../rule/engine/util/EntitiesTenantIdAsyncLoader.java | 2 +- 15 files changed, 14 insertions(+), 17 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/{alarm => id}/AlarmId.java (88%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 93754961d6..2132675f97 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index cffbce2d03..fbccaedb9e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index e2277497a5..3132aa99c7 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index a43d95975e..9c23a60c28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,6 +23,7 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java index 532842ae75..691d89cafd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java @@ -13,14 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.alarm; +package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index c367355cb0..11db1da1a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 453f1ea0be..41f8a2b630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 5867e505b6..7080722056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java index d2e906121f..819daa800b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java @@ -27,7 +27,7 @@ import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index fb4efd25d9..8d29800022 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index f02ae6b577..e0244c256e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7e0dbf3858..c19de55307 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -98,7 +98,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 40a3e8effd..31fd207c7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index a0a1c8629f..182a000b59 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 3ff25e1e8b..017bacfd72 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; From 6c4b50a380a2e3ac5d01ea7640eb40cc5df800aa Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 14:59:48 +0300 Subject: [PATCH 042/102] Implemented dedicated RE per tenant --- .../server/actors/app/AppActor.java | 28 +++++-- .../server/actors/tenant/TenantActor.java | 25 +++++-- .../server/controller/TenantController.java | 2 - .../queue/DefaultTbClusterService.java | 8 ++ .../DefaultTenantRoutingInfoService.java | 47 ++++++++++++ .../DefaultSubscriptionManagerService.java | 28 +++---- .../transport/DefaultTransportApiService.java | 19 ++++- application/src/main/resources/logback.xml | 2 + .../src/main/resources/thingsboard.yml | 27 +++---- .../ConsistentHashParitionServiceTest.java | 5 +- .../ConsistentHashPartitionService.java | 43 ++++++++--- .../queue/discovery/TenantRoutingInfo.java | 26 +++++++ .../discovery/TenantRoutingInfoService.java | 23 ++++++ .../TbQueueRemoteJsInvokeSettings.java | 12 +-- .../server/queue/util/TbCoreComponent.java | 4 + .../queue/util/TbRuleEngineComponent.java | 4 + common/queue/src/main/proto/queue.proto | 12 +++ .../common/transport/TransportService.java | 14 +++- .../service/DefaultTransportService.java | 32 ++++++++ .../TransportTenantRoutingInfoService.java | 52 +++++++++++++ .../server/dao/tenant/TenantServiceImpl.java | 35 ++++++--- .../src/main/resources/tb-mqtt-transport.yml | 74 ++++++++++++------- 22 files changed, 420 insertions(+), 102 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 145ae10d06..7e4773aa8b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -39,10 +39,13 @@ import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; +import java.util.Optional; + public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); @@ -68,7 +71,7 @@ public class AppActor extends ContextAwareActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initRuleChainsAndTenantActors(); + initTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -100,15 +103,30 @@ public class AppActor extends ContextAwareActor { return true; } - private void initRuleChainsAndTenantActors() { + private void initTenantActors() { log.info("Starting main system actor."); try { - if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - for (Tenant tenant : tenantIterator) { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (isolatedTenantId.isPresent()) { + Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); + if (tenant != null) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); + } else { + log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); + } + } else if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + for (Tenant tenant : tenantIterator) { + if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { + log.debug("[{}] Creating tenant actor", tenant.getId()); + getOrCreateTenantActor(tenant.getId()); + log.debug("[{}] Tenant actor created.", tenant.getId()); + } } } log.info("Main system actor started."); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index e54b955419..a1bf343efe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,7 +22,6 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import org.thingsboard.server.actors.ActorSystemContext; @@ -31,6 +30,7 @@ import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -47,12 +47,13 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { private final BiMap deviceActors; - private boolean isRuleEngine; + private boolean isRuleEngineForCurrentTenant; private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { @@ -69,10 +70,19 @@ public class TenantActor extends RuleChainManagerActor { public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - if (isRuleEngine) { - initRuleChains(); + + if (isRuleEngineForCurrentTenant) { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } } log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { @@ -132,8 +142,9 @@ public class TenantActor extends RuleChainManagerActor { } private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { - if (!isRuleEngine) { + if (!isRuleEngineForCurrentTenant) { log.warn("RECEIVED INVALID MESSAGE: {}", msg); + return; } TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { @@ -167,7 +178,7 @@ public class TenantActor extends RuleChainManagerActor { } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - if (isRuleEngine) { + if (isRuleEngineForCurrentTenant) { ActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index abd6bf6804..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -76,7 +76,6 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); - tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -96,7 +95,6 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5851fd919c..fa6aa9e886 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -54,6 +54,14 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java new file mode 100644 index 0000000000..aae3cef0ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") +public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { + + private final TenantService tenantService; + + public DefaultTenantRoutingInfoService(TenantService tenantService) { + this.tenantService = tenantService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + Tenant tenant = tenantService.findTenantById(tenantId); + if (tenant != null) { + return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); + } else { + throw new RuntimeException("Tenant not found!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 0d151c3f46..e841acae12 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -173,19 +173,21 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - Set removedPartitions = new HashSet<>(currentPartitions); - removedPartitions.removeAll(partitionChangeEvent.getPartitions()); - - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set subs = partitionedSubscriptions.remove(partition); - if (subs != null) { - subs.forEach(this::removeSubscriptionFromEntityMap); - } - }); + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 4ff647b4d9..7456ca7662 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -24,8 +24,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -34,14 +34,18 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -59,6 +63,10 @@ public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); + //TODO: Constructor dependencies; + @Autowired + private TenantService tenantService; + @Autowired private DeviceService deviceService; @@ -87,6 +95,8 @@ public class DefaultTransportApiService implements TransportApiService { return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } @@ -129,6 +139,13 @@ public class DefaultTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } + private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); + ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); + return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() + .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) + .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); + } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 8389fa5d71..08a976592d 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index affd7e7908..a2a06b57d0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -437,18 +437,6 @@ js: print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" # Maximum time in seconds for black listed function to stay in the list. @@ -579,6 +567,19 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -586,7 +587,7 @@ queue: stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: # TODO 2.5: specify correct ENV variable names. + queues: - name: "Main" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 2634d5af99..3f7619ed1e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +50,7 @@ public class ConsistentHashParitionServiceTest { private ConsistentHashPartitionService clusterRoutingService; private TbServiceInfoProvider discoveryService; + private TenantRoutingInfoService routingInfoService; private ApplicationEventPublisher applicationEventPublisher; private String hashFunctionName = "murmur3_128"; @@ -59,7 +61,8 @@ public class ConsistentHashParitionServiceTest { public void setup() throws Exception { discoveryService = mock(TbServiceInfoProvider.class); applicationEventPublisher = mock(ApplicationEventPublisher.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, applicationEventPublisher); + routingInfoService = mock(TenantRoutingInfoService.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 0552aa05c6..dfd1437b5f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -60,11 +60,12 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; + private final TenantRoutingInfoService tenantRoutingInfoService; private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); - //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. - private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); private Map tbCoreNotificationTopics = new HashMap<>(); @@ -73,8 +74,9 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { this.serviceInfoProvider = serviceInfoProvider; + this.tenantRoutingInfoService = tenantRoutingInfoService; this.applicationEventPublisher = applicationEventPublisher; } @@ -122,10 +124,10 @@ public class ConsistentHashPartitionService implements PartitionService { addNode(circles, other); } ConcurrentMap> oldPartitions = myPartitions; - TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); + TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { - ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myTenantId); + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); if (currentService.equals(serviceInfo)) { @@ -247,7 +249,30 @@ public class ConsistentHashPartitionService implements PartitionService { } private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { - return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceQueue.getType()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return false; + } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + synchronized (tenantRoutingInfoMap) { + routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + tenantRoutingInfoMap.put(tenantId, routingInfo); + } + } + } + if (routingInfo == null) { + throw new RuntimeException("Tenant not found!"); + } + switch (serviceQueue.getType()) { + case TB_CORE: + return routingInfo.isIsolatedTbCore(); + case TB_RULE_ENGINE: + return routingInfo.isIsolatedTbRuleEngine(); + default: + return false; + } } private void logServiceInfo(TransportProtos.ServiceInfo server) { @@ -265,12 +290,6 @@ public class ConsistentHashPartitionService implements PartitionService { private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); - if (!tenantId.isNullUid()) { - isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); - for (String serviceType : instance.getServiceTypesList()) { - isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); - } - } for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java new file mode 100644 index 0000000000..07c4c2d9c5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class TenantRoutingInfo { + private final TenantId tenantId; + private final boolean isolatedTbCore; + private final boolean isolatedTbRuleEngine; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java new file mode 100644 index 0000000000..685a36df30 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantRoutingInfoService { + + TenantRoutingInfo getRoutingInfo(TenantId tenantId); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index cdad407b19..cd492e87f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -22,21 +22,21 @@ import org.springframework.stereotype.Component; @Data @Component public class TbQueueRemoteJsInvokeSettings { - @Value("${js.remote.request_topic}") + @Value("${queue.js.request_topic}") private String requestTopic; - @Value("${js.remote.response_topic_prefix}") + @Value("${queue.js.response_topic_prefix}") private String responseTopic; - @Value("${js.remote.max_pending_requests}") + @Value("${queue.js.max_pending_requests}") private long maxPendingRequests; - @Value("${js.remote.response_poll_interval}") + @Value("${queue.js.response_poll_interval}") private int responsePollInterval; - @Value("${js.remote.response_auto_commit_interval}") + @Value("${queue.js.response_auto_commit_interval}") private int autoCommitInterval; - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index c42b9b9f7d..ca2316e174 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -17,6 +17,10 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public @interface TbCoreComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java index 855bebb19e..3b99c94dea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -17,6 +17,10 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") public @interface TbRuleEngineComponent { } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index e36c4e36a9..3db8495d45 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -158,6 +158,16 @@ message GetOrCreateDeviceFromGatewayResponseMsg { DeviceInfoProto deviceInfo = 1; } +message GetTenantRoutingInfoRequestMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; +} + +message GetTenantRoutingInfoResponseMsg { + bool isolatedTbCore = 1; + bool isolatedTbRuleEngine = 2; +} + message SessionCloseNotificationProto { string message = 1; } @@ -359,12 +369,14 @@ message TransportApiRequestMsg { ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; } /* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; } /* Messages that are handled by ThingsBoard Core Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index b0772de945..2af55c9206 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.common.transport; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -35,14 +39,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { + GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); + void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -62,7 +68,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index d107bbbeeb..7b2f157cb4 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -43,6 +44,8 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.gen.transport.TransportProtos; @@ -61,6 +64,7 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -194,6 +198,34 @@ public class DefaultTransportService implements TransportService { sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } + @Override + public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { + TbProtoQueueMsg protoMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getGetTenantRoutingInfoResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + + TransportProtos.GetTenantRoutingInfoRequestMsg msg = TransportProtos.GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + TransportProtos.GetTenantRoutingInfoResponseMsg routingInfo = response.getValue().getGetTenantRoutingInfoResponseMsg(); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + @Override public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java new file mode 100644 index 0000000000..f2534a64c9 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { + + private TransportService transportService; + + @Lazy + @Autowired + public void setTransportService(TransportService transportService) { + this.transportService = transportService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 9453fdb4c5..eb916cdb83 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; @@ -126,7 +127,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -140,19 +141,31 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - }; + + @Override + protected void validateUpdate(TenantId tenantId, Tenant tenant) { + Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing tenant!"); + } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); + } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new DataValidationException("Can't update isolatedTbCore property!"); + } + } + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 395f1d09e0..c421c43e33 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -111,12 +111,25 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -124,32 +137,39 @@ queue: stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: # TODO 2.5: specify correct ENV variable names. - - - name: "Main" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - - name: "HighPriority" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: + # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" From 006fcd03ed09eff402074c33bc2426f7b1d0f706 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 15:01:20 +0300 Subject: [PATCH 043/102] Improved configuration for HTTP and CoAP transport --- .../src/main/resources/tb-coap-transport.yml | 135 +++++++++++++++--- .../src/main/resources/tb-http-transport.yml | 135 +++++++++++++++--- 2 files changed, 230 insertions(+), 40 deletions(-) diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 9b3572c7d1..70d1ff8fbe 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,24 +41,119 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 4097bce831..5ddb7c0553 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,24 +42,119 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file From d380ac710646a2aea8693ea14aba1eaa4b9f4994 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 18:33:11 +0300 Subject: [PATCH 044/102] Device -> Server RPC implementation and improvements --- .../server/actors/ActorSystemContext.java | 4 - .../server/actors/device/DeviceActor.java | 9 -- .../device/DeviceActorMessageProcessor.java | 108 +++++-------- .../queue/DefaultTbClusterService.java | 6 + .../service/queue/TbClusterService.java | 3 + .../rpc/DefaultTbRuleEngineRpcService.java | 14 +- .../rpc/ToServerRpcResponseActorMsg.java | 47 ------ .../DefaultTbCoreToTransportService.java | 27 ++-- .../transport/TbCoreToTransportService.java | 6 +- .../src/main/resources/thingsboard.yml | 22 +-- .../server/common/msg/MsgType.java | 2 - .../DeviceActorClientSideRpcTimeoutMsg.java | 33 ---- .../ConsistentHashPartitionService.java | 5 - .../queue/discovery/PartitionService.java | 2 - .../queue/kafka/TBKafkaProducerTemplate.java | 2 +- common/queue/src/main/proto/queue.proto | 18 +-- .../transport/coap/CoapTransportResource.java | 2 + .../transport/http/DeviceApiController.java | 2 + .../transport/mqtt/MqttTransportHandler.java | 2 + .../mqtt/session/GatewayDeviceSessionCtx.java | 2 + .../service/DefaultTransportService.java | 144 +++++++++++------- .../service/RpcRequestMetadata.java} | 16 +- .../rule/engine/api/RuleEngineRpcService.java | 4 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 10 +- .../rpc/TbSendRpcReplyNodeConfiguration.java | 24 ++- .../src/main/resources/tb-coap-transport.yml | 18 +-- .../src/main/resources/tb-http-transport.yml | 18 +-- .../src/main/resources/tb-mqtt-transport.yml | 36 +++-- 28 files changed, 266 insertions(+), 320 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java rename common/{message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java => transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java} (76%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index d31cb62b9b..7822438724 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -270,10 +270,6 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; - @Value("${actors.client_side_rpc.timeout}") - @Getter - private long clientSideRpcTimeout; - @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 3a15240f33..256ffd0b30 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,11 +22,8 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -72,15 +69,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; - case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); - break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; - case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: - processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); - break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 83d269ef6d..437ed456e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -36,12 +36,11 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; @@ -50,16 +49,19 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotifica import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.Nullable; @@ -91,7 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; - private final Map toServerRpcPendingMap; private int rpcSeq = 0; private String deviceName; @@ -106,7 +107,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); - this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -142,7 +142,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (TransportProtos.SessionType.SYNC == value.getType()) { + if (SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -177,10 +177,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - TransportProtos.SessionType sessionType = getSessionType(sessionId); + SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == TransportProtos.SessionType.SYNC) { + if (sessionType == SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -188,7 +188,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == TransportProtos.SessionType.ASYNC) { + if (sessionType == SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -297,46 +297,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { -// UUID sessionId = getSessionId(sessionInfo); -// JsonObject json = new JsonObject(); -// json.addProperty("method", request.getMethodName()); -// json.add("params", JsonUtils.parse(request.getParams())); -// -// TbMsgMetaData requestMetaData = defaultMetaData.copy(); -// requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); -// TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); -// context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); -// -// scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); -// toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); - } - - private TransportProtos.SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; - } - - void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); - if (data != null) { - log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(msg.getId()).setError("timeout").build() - , data.getSessionId(), data.getNodeId()); - } - } - - void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { - int requestId = msg.getMsg().getRequestId(); - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() - , data.getSessionId(), data.getNodeId()); - } else { - log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); - } + private SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -399,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -420,7 +382,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -444,7 +406,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } @@ -462,10 +424,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -488,7 +450,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); @@ -504,7 +466,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); @@ -512,7 +474,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); @@ -520,15 +482,15 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); systemContext.getTbCoreToTransportService().process(nodeId, msg); } - private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); @@ -584,9 +546,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -595,11 +557,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); - TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); + SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -617,27 +579,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() + SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index fa6aa9e886..f93657427d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import java.util.HashSet; import java.util.Set; +import java.util.UUID; @Service @Slf4j @@ -100,7 +101,12 @@ public class DefaultTbClusterService implements TbClusterService { response.getResponse().ifPresent(builder::setResponse); ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + } + @Override + public void onToTransportMsg(String serviceId, ToTransportMsg response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), null); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java index 952b88e4d3..b70a31532e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; public interface TbClusterService { @@ -32,6 +33,8 @@ public interface TbClusterService { void onToRuleEngineMsg(String targetServiceId, FromDeviceRpcResponse response); + void onToTransportMsg(String targetServiceId, ToTransportMsg response); + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index c383da3a79..909690ff6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -52,6 +53,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi private final TbClusterService clusterService; private final TbServiceInfoProvider serviceInfoProvider; + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); private Optional tbCoreRpcService; @@ -85,8 +87,16 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi } @Override - public void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body) { -// TODO 2.5 + public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { + TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId) + .setPayload(body).build(); + TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setToServerResponse(responseMsg) + .build(); + clusterService.onToTransportMsg(serviceId, msg); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java deleted file mode 100644 index 570112dab3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.ToString; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; - -/** - * Created by ashvayka on 16.04.18. - */ -@ToString -@RequiredArgsConstructor -public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { - - @Getter - private final TenantId tenantId; - - @Getter - private final DeviceId deviceId; - - @Getter - private final ToServerRpcResponseMsg msg; - - @Override - public MsgType getMsgType() { - return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 8f29c46466..c9e37e6791 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -18,13 +18,14 @@ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -38,28 +39,26 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @TbCoreComponent public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + private final PartitionService partitionService; private final TbQueueProducer> tbTransportProducer; - @Value("${queue.transport.notifications_topic}") - private String notificationsTopic; - - public DefaultTbCoreToTransportService(TbQueueProducerProvider tbQueueProducerProvider) { + public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { + this.partitionService = partitionService; this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); } @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { + public void process(String nodeId, ToTransportMsg msg) { process(nodeId, msg, null, null); } @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; + public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); - tbTransportProducer.send(TopicPartitionInfo.builder().topic(topic).build(), queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); + tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private static class QueueCallbackAdaptor implements TbQueueCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 5b11edb5f0..e0d5e2121a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -15,14 +15,14 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import java.util.function.Consumer; public interface TbCoreToTransportService { - void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg); + void process(String nodeId, ToTransportMsg msg); - void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a2a06b57d0..b5e875e07b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -251,8 +251,6 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -458,6 +456,8 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -552,15 +552,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -569,9 +569,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -581,7 +581,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -589,7 +589,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -604,7 +604,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -620,7 +620,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 47e09b812f..0345355aa3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -89,8 +89,6 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, - DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, - /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java deleted file mode 100644 index 5d47425e43..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.timeout; - -import org.thingsboard.server.common.msg.MsgType; - -/** - * @author Andrew Shvayka - */ -public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { - - public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { - super(id, timeout); - } - - @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index dfd1437b5f..df25ea3ba9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -196,11 +196,6 @@ public class ConsistentHashPartitionService implements PartitionService { } } - @Override - public Set getIsolatedTenants(ServiceType serviceType) { - throw new RuntimeException("Not Implemented!"); - } - private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 7c6900d8a8..e3b3e76b80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -47,8 +47,6 @@ public interface PartitionService { */ Set getAllServiceIds(ServiceType serviceType); - Set getIsolatedTenants(ServiceType serviceType); - /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 2fbfa7d3df..2d82ed275d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -79,7 +79,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure", exception); + log.warn("Producer template failure: {}", exception.getMessage(), exception); } } }); diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 3db8495d45..dac64a3478 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -245,16 +245,6 @@ message TransportToRuleEngineMsg { ToServerRpcRequestMsg toServerRPCCallRequest = 5; } -message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} - /** * TB Core Data Structures */ @@ -410,5 +400,11 @@ message ToRuleEngineNotificationMsg { /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 9d2cff5c09..82678fde5d 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,6 +303,8 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index c8e7f76450..c2f171e39b 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -221,6 +221,8 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index fb509e91af..7295cb786b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -507,6 +507,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index 299f5449d9..b4b5f98238 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,6 +44,8 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) + .setDeviceName(deviceInfo.getDeviceName()) + .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 7b2f157cb4..389f50009f 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,36 +18,25 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.transport.util.JsonUtils; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueFactory; +import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -55,11 +44,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -90,6 +91,8 @@ public class DefaultTransportService implements TransportService { private long sessionInactivityTimeout; @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Value("${transport.client_side_rpc.timeout:60000}") + private long clientSideRpcTimeout; @Value("${queue.transport.poll_interval}") private int notificationsPollDuration; @@ -97,6 +100,7 @@ public class DefaultTransportService implements TransportService { private final TbTransportQueueFactory queueProvider; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -106,15 +110,17 @@ public class DefaultTransportService implements TransportService { protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; - private ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); //TODO: Implement cleanup of this maps. - private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; this.partitionService = partitionService; @@ -134,7 +140,8 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); - transportNotificationsConsumer.subscribe(); + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); + transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { @@ -145,10 +152,7 @@ public class DefaultTransportService implements TransportService { } records.forEach(record -> { try { - ToTransportMsg toTransportMsg = record.getValue(); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } + processToTransportMsg(record.getValue()); } catch (Throwable e) { log.warn("Failed to process the notification.", e); } @@ -195,7 +199,7 @@ public class DefaultTransportService implements TransportService { @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } @Override @@ -210,22 +214,6 @@ public class DefaultTransportService implements TransportService { } } - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - - TransportProtos.GetTenantRoutingInfoRequestMsg msg = TransportProtos.GetTenantRoutingInfoRequestMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .build(); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); - try { - TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); - TransportProtos.GetTenantRoutingInfoResponseMsg routingInfo = response.getValue().getGetTenantRoutingInfoResponseMsg(); - return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - @Override public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); @@ -253,7 +241,7 @@ public class DefaultTransportService implements TransportService { @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscriptionInfo(msg).build(), callback); @@ -340,13 +328,51 @@ public class DefaultTransportService implements TransportService { } } - //TODO 2.5: Need to handle timeouts on the transport level and not on the Device Actor Level. + private void processTimeout(String requestId) { + RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + SessionMetaData md = sessions.get(data.getSessionId()); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + TransportProtos.ToServerRpcResponseMsg responseMsg = + TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(data.getRequestId()) + .setError("timeout").build(); + listener.onToServerRpcResponse(responseMsg); + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + log.debug("[{}] Missing session.", data.getSessionId()); + } + } + } + @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); -// sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). -// setToServerRPCCallRequest(msg).build(), new TransportTbQueueCallback(callback)); + UUID sessionId = toSessionId(sessionInfo); + TenantId tenantId = getTenantId(sessionInfo); + DeviceId deviceId = getDeviceId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", msg.getMethodName()); + json.add("params", JsonUtils.parse(msg.getParams())); + + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("requestId", Integer.toString(msg.getRequestId())); + metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); + metaData.putValue("sessionId", sessionId.toString()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + + String requestId = sessionId + "-" + msg.getRequestId(); + toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); + schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); } } @@ -364,7 +390,7 @@ public class DefaultTransportService implements TransportService { } private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toId(sessionInfo); + UUID sessionId = toSessionId(sessionInfo); SessionMetaData sessionMetaData = sessions.get(sessionId); if (sessionMetaData != null) { sessionMetaData.updateLastActivityTime(); @@ -377,7 +403,7 @@ public class DefaultTransportService implements TransportService { sessions.forEach((uuid, sessionMD) -> { if (sessionMD.getLastActivityTime() < expTime) { if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); } process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); sessions.remove(uuid); @@ -407,7 +433,7 @@ public class DefaultTransportService implements TransportService { @Override public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toId(sessionInfo), currentSession); + sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); @@ -419,18 +445,18 @@ public class DefaultTransportService implements TransportService { @Override public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toId(sessionInfo)); + SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); if (currentSession != null && currentSession.hasScheduledFuture()) { log.debug("Stopping scheduler to avoid resending response if request has been ack."); currentSession.getScheduledFuture().cancel(false); } - sessions.remove(toId(sessionInfo)); + sessions.remove(toSessionId(sessionInfo)); } @Override public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } if (!rateLimitEnabled) { return true; @@ -442,7 +468,7 @@ public class DefaultTransportService implements TransportService { callback.onError(new TbRateLimitsException(EntityType.TENANT)); } if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); + log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); } return false; } @@ -453,7 +479,7 @@ public class DefaultTransportService implements TransportService { callback.onError(new TbRateLimitsException(EntityType.DEVICE)); } if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); + log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); } return false; } @@ -461,7 +487,7 @@ public class DefaultTransportService implements TransportService { return true; } - protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { + protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); SessionMetaData md = sessions.get(sessionId); if (md != null) { @@ -480,6 +506,8 @@ public class DefaultTransportService implements TransportService { listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); } if (toSessionMsg.hasToServerResponse()) { + String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); + toServerRpcPendingMap.remove(requestId); listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); } }); @@ -492,7 +520,7 @@ public class DefaultTransportService implements TransportService { } } - protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java similarity index 76% rename from common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java index d06046fd71..2c5da0d3ea 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java @@ -13,20 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.core; +package org.thingsboard.server.common.transport.service; import lombok.Data; -/** - * @author Andrew Shvayka - */ -@Data -public class ToServerRpcResponseMsg { +import java.util.UUID; +@Data +public class RpcRequestMetadata { + private final UUID sessionId; private final int requestId; - private final String data; - - public boolean isSuccess() { - return true; - } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 5b76ecf624..baac594a17 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; + +import java.util.UUID; import java.util.function.Consumer; /** @@ -23,7 +25,7 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 393ef0bd7c..145e73d450 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.UUID; + @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -50,15 +52,21 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { + String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); + String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); + } else if (StringUtils.isEmpty(serviceIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); + } else if (StringUtils.isEmpty(sessionIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index d01f28f136..bd3a3cdfb2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,18 +16,40 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; +import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { + public static final String SERVICE_ID = "serviceId"; + public static final String SESSION_ID = "sessionId"; + public static final String REQUEST_ID = "requestId"; + + private String serviceIdMetaDataAttribute; + private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setRequestIdMetaDataAttribute("requestId"); + configuration.setServiceIdMetaDataAttribute(SERVICE_ID); + configuration.setSessionIdMetaDataAttribute(SESSION_ID); + configuration.setRequestIdMetaDataAttribute(REQUEST_ID); return configuration; } + + public String getServiceIdMetaDataAttribute() { + return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; + } + + public String getSessionIdMetaDataAttribute() { + return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; + } + + public String getRequestIdMetaDataAttribute() { + return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; + } } + diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 70d1ff8fbe..b94148f608 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -81,15 +81,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -98,9 +98,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -110,7 +110,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -118,7 +118,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -133,7 +133,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -149,7 +149,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 5ddb7c0553..0fa2788cac 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -82,15 +82,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -99,9 +99,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -111,7 +111,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -119,7 +119,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -134,7 +134,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -150,7 +150,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index c421c43e33..747427caf8 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # MQTT server parameters transport: @@ -102,15 +112,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -119,9 +129,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -131,7 +141,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -139,7 +149,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -154,7 +164,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -170,7 +180,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: From e83d69c40ced610b0cfc7db05424b8b4bb43e73f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 20:16:51 +0300 Subject: [PATCH 045/102] isolated tenant improvements --- .../server/controller/TenantController.java | 11 +++++++++++ .../service/install/SqlDatabaseUpgradeService.java | 5 ++++- ui/src/app/tenant/tenant-fieldset.tpl.html | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 9def943e88..e9a793fb34 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -28,6 +28,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; @@ -72,6 +73,16 @@ public class TenantController extends BaseController { try { boolean newTenant = tenant.getId() == null; + if (!newTenant) { + Tenant oldTenant = tenantService.findTenantById(tenant.getTenantId()); + if (oldTenant.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new ThingsboardException("Field isolatedTbCore from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + if (oldTenant.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new ThingsboardException("Field isolatedTbRuleEngine from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } + Operation operation = newTenant ? Operation.CREATE : Operation.WRITE; accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index bcbd77ef28..f673f498fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,7 +221,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } - conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + try { + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index 35b3dee289..b75ea30f06 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -47,14 +47,14 @@ - {{'tenant.isolated-tb-core' | translate}}
{{'tenant.isolated-tb-core-details' | translate}} - {{'tenant.isolated-tb-rule-engine' | translate}}
{{'tenant.isolated-tb-rule-engine-details' | translate}} From d024ba56e89fad42842a5c9b9aafacb865d6466e Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 21:33:07 +0300 Subject: [PATCH 046/102] removed tenant validation from controller --- .../server/controller/TenantController.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index e9a793fb34..623fbb65c9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -73,16 +73,6 @@ public class TenantController extends BaseController { try { boolean newTenant = tenant.getId() == null; - if (!newTenant) { - Tenant oldTenant = tenantService.findTenantById(tenant.getTenantId()); - if (oldTenant.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { - throw new ThingsboardException("Field isolatedTbCore from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - if (oldTenant.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { - throw new ThingsboardException("Field isolatedTbRuleEngine from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - } - Operation operation = newTenant ? Operation.CREATE : Operation.WRITE; accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, From 2e122d86263aba1160b806d3229533f28e322392 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 21:34:32 +0300 Subject: [PATCH 047/102] refactored --- .../java/org/thingsboard/server/controller/TenantController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 623fbb65c9..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; From 63bac58ae84d31f3976ab447c088741fbfc50aeb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 08:27:44 +0300 Subject: [PATCH 048/102] Implementation of statistics counters --- .../server/actors/ActorSystemContext.java | 4 - .../actors/ruleChain/DefaultTbContext.java | 11 +- .../RuleChainActorMessageProcessor.java | 10 +- .../actors/service/DefaultActorService.java | 92 ------------- .../server/controller/BaseController.java | 3 +- .../server/controller/DeviceController.java | 6 +- .../controller/TelemetryController.java | 8 +- .../queue/DefaultTbClusterService.java | 113 ++++++++++++---- .../queue/DefaultTbCoreConsumerService.java | 24 ++-- .../service/queue/TbClusterService.java | 22 ++- .../service/queue/TbCoreConsumerStats.java | 25 +++- .../rpc/DefaultTbCoreDeviceRpcService.java | 5 +- .../rpc/DefaultTbRuleEngineRpcService.java | 7 +- .../state/DefaultDeviceStateService.java | 21 +-- .../DefaultSubscriptionManagerService.java | 5 +- .../DefaultTbLocalSubscriptionService.java | 12 +- .../DefaultTelemetrySubscriptionService.java | 23 ++-- .../TbTransportQueueProducerProvider.java | 2 - .../rule/engine/api/TbContext.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 1 - .../rule/engine/rest/TbHttpClient.java | 23 +--- .../engine/rest/TbRedisQueueProcessor.java | 126 ------------------ .../rule/engine/rest/TbRestApiCallNode.java | 19 +-- 23 files changed, 186 insertions(+), 378 deletions(-) delete mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7822438724..fb4ff77359 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -159,10 +159,6 @@ public class ActorSystemContext { @Getter private TbClusterService clusterService; - @Autowired - @Getter - private TbQueueProducerProvider producerProvider; - @Autowired @Getter private TimeseriesService tsService; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 0e8033fac2..13f7b1122d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,10 +17,8 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; @@ -36,7 +34,6 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; @@ -45,8 +42,8 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; @@ -62,11 +59,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -136,7 +131,7 @@ class DefaultTbContext implements TbContext { .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override @@ -193,7 +188,7 @@ class DefaultTbContext implements TbContext { if (failureMessage != null) { msg.setFailureMessage(failureMessage); } - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg.build()), new SimpleTbQueueCallback(onSuccess, onFailure)); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index dd0a515a33..a374e590b5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -44,10 +44,9 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.service.queue.TbClusterService; import java.util.ArrayList; import java.util.Collections; @@ -68,7 +67,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final TbQueueProducer> producer; + private final TbClusterService clusterService; private String ruleChainName; private RuleNodeId firstId; @@ -84,7 +83,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.producer = systemContext.getProducerProvider().getRuleEngineMsgProducer(); + this.clusterService = systemContext.getClusterService(); } @Override @@ -255,7 +254,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(newMsg.getId(), toQueueMsg), callbackWrapper); + clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); } private boolean contains(Set relationTypes, String type) { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 537e897583..7b04e82786 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -97,96 +97,4 @@ public class DefaultActorService implements ActorService { } } - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - //TODO 2.5 - private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); - private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); - - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int sent = sentClusterMsgs.getAndSet(0); - int received = receivedClusterMsgs.getAndSet(0); - if (sent > 0 || received > 0) { - log.info("Cluster msgs sent [{}] received [{}]", sent, received); - } - } - } - - //TODO 2.5 -// @Override -// public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { -// if (statsEnabled) { -// receivedClusterMsgs.incrementAndGet(); -// } -// ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); -// if (log.isDebugEnabled()) { -// log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); -// log.info("MSG: {}", msg); -// } -// switch (msg.getMessageType()) { -// case CLUSTER_ACTOR_MESSAGE: -// java.util.Optional decodedMsg = actorContext.getEncodingService() -// .decode(msg.getPayload().toByteArray()); -// if (decodedMsg.isPresent()) { -// appActor.tell(decodedMsg.get(), ActorRef.noSender()); -// } else { -// log.error("Error during decoding cluster proto message"); -// } -// break; -// case TO_ALL_NODES_MSG: -// //TODO -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: -// actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: -// actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: -// actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: -// actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: -// actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TRANSACTION_SERVICE_MESSAGE: -// actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); -// break; -// } -// } -// @Override -// public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { -// if (statsEnabled) { -// sentClusterMsgs.incrementAndGet(); -// } -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } -// -// @Override -// public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { -// if (statsEnabled) { -// sentClusterMsgs.incrementAndGet(); -// } -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } -// @Override -// public void onBroadcastMsg(RpcBroadcastMsg msg) { -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index fbccaedb9e..73b25d4de2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -668,7 +667,7 @@ public abstract class BaseController { } } TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - tbClusterService.onToRuleEngineMsg(user.getTenantId(), entityId, tbMsg); + tbClusterService.pushMsgToRuleEngine(user.getTenantId(), entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index afa394b98c..ca61ecc8e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -99,8 +99,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - tbClusterService.onToCoreMsg(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), - savedDevice.getId(), savedDevice.getName(), savedDevice.getType())); + tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -254,7 +254,7 @@ public class DeviceController extends BaseController { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - tbClusterService.onToCoreMsg(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId())); + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); logEntityAction(device.getId(), device, device.getCustomerId(), diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index d0c4be2386..8352e3f457 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -364,8 +364,8 @@ public class TelemetryController extends BaseController { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify), null); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -399,8 +399,8 @@ public class TelemetryController extends BaseController { logAttributesUpdated(user, entityId, scope, attributes, null); if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes), null); } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f93657427d..f0054771d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.EntityType; @@ -27,7 +29,13 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,12 +46,22 @@ import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import java.util.HashSet; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; @Service @Slf4j public class DefaultTbClusterService implements TbClusterService { - protected TbQueueProducerProvider producerProvider; + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger toCoreMsgs = new AtomicInteger(0); + private final AtomicInteger toCoreNfs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); + private final AtomicInteger toTransportNfs = new AtomicInteger(0); + + private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; private final DataDecodingEncodingService encodingService; @@ -54,33 +72,29 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { - if (tenantId.isNullUid()) { - if (entityId.getEntityType().equals(EntityType.TENANT)) { - tenantId = new TenantId(entityId.getId()); - } else { - log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); - return; - } - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toCoreMsgs.incrementAndGet(); } @Override - public void onToCoreMsg(ToDeviceActorNotificationMsg msg) { + public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); byte[] msgBytes = encodingService.encode(msg); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), null); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); + toCoreMsgs.incrementAndGet(); } @Override - public void onToCoreMsg(String serviceId, FromDeviceRpcResponse response) { + public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) @@ -88,11 +102,37 @@ public class DefaultTbClusterService implements TbClusterService { .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); response.getResponse().ifPresent(builder::setResponse); ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toCoreNfs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); } @Override - public void onToRuleEngineMsg(String serviceId, FromDeviceRpcResponse response) { + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) @@ -100,13 +140,15 @@ public class DefaultTbClusterService implements TbClusterService { .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); response.getResponse().ifPresent(builder::setResponse); ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toRuleEngineNfs.incrementAndGet(); } @Override - public void onToTransportMsg(String serviceId, ToTransportMsg response) { + public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); - producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), null); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); + toTransportNfs.incrementAndGet(); } @Override @@ -120,12 +162,13 @@ public class DefaultTbClusterService implements TbClusterService { TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { - TbQueueProducer> toCoreProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfs.incrementAndGet(); } // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); @@ -134,6 +177,22 @@ public class DefaultTbClusterService implements TbClusterService { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + } + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int toCoreMsgCnt = toCoreMsgs.getAndSet(0); + int toCoreNfsCnt = toCoreNfs.getAndSet(0); + int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); + int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); + int toTransportNfsCnt = toTransportNfs.getAndSet(0); + if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { + log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", + toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); + } } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 263e6eab24..c943e4c4d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -183,21 +183,24 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbCallback callback) { - ToCoreNotificationMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreMsg.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); - } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + ToCoreNotificationMsg toCoreNotification = msg.getValue(); + if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreNotification.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); + } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } + if (statsEnabled) { + stats.log(toCoreNotification); + } } private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { @@ -246,6 +249,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService 0) { - log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]", + log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + " deviceState [{}] subMgr [{}] coreNfs [{}]", total, sessionEventCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) + , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index 6d70056af0..8a1423f97c 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.rpc; import akka.actor.ActorRef; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -147,7 +146,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); } } else { - clusterService.onToRuleEngineMsg(originServiceId, response); + clusterService.pushNotificationToRuleEngine(originServiceId, response, null); } } @@ -170,7 +169,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { try { TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - clusterService.onToRuleEngineMsg(msg.getTenantId(), msg.getDeviceId(), tbMsg); + clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index 909690ff6c..0ec730b7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -22,7 +22,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -96,7 +95,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(responseMsg) .build(); - clusterService.onToTransportMsg(serviceId, msg); + clusterService.pushNotificationToTransport(serviceId, msg, null); } @Override @@ -148,7 +147,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi } } else { log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); - clusterService.onToCoreMsg(rpcMsg); + clusterService.pushMsgToCore(rpcMsg, null); } } @@ -160,7 +159,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); } } else { - clusterService.onToCoreMsg(originServiceId, response); + clusterService.pushNotificationToCore(originServiceId, response, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 87f46f2072..1e3ab73a26 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -29,7 +29,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -56,8 +55,8 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -106,7 +105,7 @@ public class DefaultDeviceStateService implements DeviceStateService { private final DeviceService deviceService; private final AttributesService attributesService; private final TimeseriesService tsService; - private final TbQueueProducerProvider producerProvider; + private final TbClusterService clusterService; private final PartitionService partitionService; private TelemetrySubscriptionService tsSubService; @@ -137,12 +136,12 @@ public class DefaultDeviceStateService implements DeviceStateService { public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, PartitionService partitionService) { + TbClusterService clusterService, PartitionService partitionService) { this.tenantService = tenantService; this.deviceService = deviceService; this.attributesService = attributesService; this.tsService = tsService; - this.producerProvider = producerProvider; + this.clusterService = clusterService; this.partitionService = partitionService; } @@ -413,8 +412,6 @@ public class DefaultDeviceStateService implements DeviceStateService { } private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - log.trace("[{}][{}] Device is monitored on partition: {}", tenantId, deviceId, tpi); TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -424,8 +421,7 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setUpdated(updated); builder.setDeleted(deleted); TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), - TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); + clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { @@ -497,12 +493,7 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state)); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index e841acae12..67c81ac06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -245,8 +245,9 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - clusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))); + clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) + , null); } } callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 12c5e206e3..d57067fa28 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -28,16 +28,14 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -70,19 +68,17 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer private PartitionService partitionService; @Autowired - private TbQueueProducerProvider producerProvider; + private TbClusterService clusterService; @Autowired @Lazy private SubscriptionManagerService subscriptionManagerService; private ExecutorService wsCallBackExecutor; - private TbQueueProducer> toCoreProducer; @PostConstruct public void initExecutor() { wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -140,7 +136,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); } } @@ -181,7 +177,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); } } else { log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index b3c728a25b..665a787e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,17 +32,15 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -69,22 +67,20 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private final AttributesService attrService; private final TimeseriesService tsService; - private final TbQueueProducerProvider producerProvider; + private final TbClusterService clusterService; private final PartitionService partitionService; private Optional subscriptionManagerService; - private TbQueueProducer> toCoreProducer; - private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, + TbClusterService clusterService, PartitionService partitionService) { this.attrService = attrService; this.tsService = tsService; - this.producerProvider = producerProvider; + this.clusterService = clusterService; this.partitionService = partitionService; } @@ -97,7 +93,6 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); - toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -172,7 +167,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } @@ -186,7 +181,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 0ebd59ec24..bced3f83ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -23,13 +23,11 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.PostConstruct; -//TODO 2.5 Maybe remove this service if it is not used. @Service @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 4299aeb867..daab510102 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -211,7 +211,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); - //TODO 2.5: - need to remove this. + @Deprecated RedisTemplate getRedisTemplate(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 10e664bc16..19d8515455 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -75,7 +75,6 @@ public class TbMsgCountNode implements TbNode { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - //TODO 2.5: Callback? TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 5b30f4792c..ba860c103b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -84,7 +83,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + void processMessage(TbContext ctx, TbMsg msg) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -95,13 +94,6 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { - if (config.isUseRedisQueueForMsgPersistence()) { - if (throwable instanceof HttpClientErrorException) { - processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); - } else { - queueProcessor.pushOnFailure(msg); - } - } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -109,15 +101,9 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { - if (config.isUseRedisQueueForMsgPersistence()) { - queueProcessor.resetCounter(); - } TbMsg next = processResponse(ctx, msg, responseEntity); ctx.tellSuccess(next); } else { - if (config.isUseRedisQueueForMsgPersistence()) { - processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); - } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -183,11 +169,4 @@ class TbHttpClient { } } - private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { - if (statusCode.is4xxClientError()) { - log.warn("[{}] Client error during message delivering!", msg); - } else { - queueProcessor.pushOnFailure(msg); - } - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java deleted file mode 100644 index 015cced6c6..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.rest; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.redis.core.ListOperations; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Data -@Slf4j -class TbRedisQueueProcessor { - - private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; - - private final TbContext ctx; - private final TbHttpClient httpClient; - private final ExecutorService executor; - private final ListOperations listOperations; - private final String redisKey; - private final boolean trimQueue; - private final int maxQueueSize; - - private AtomicInteger failuresCounter; - private Future future; - - TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { - this.ctx = ctx; - this.httpClient = httpClient; - this.executor = Executors.newSingleThreadExecutor(); - this.listOperations = ctx.getRedisTemplate().opsForList(); - this.redisKey = constructRedisKey(); - this.trimQueue = trimQueue; - this.maxQueueSize = maxQueueSize; - init(); - } - - private void init() { - failuresCounter = new AtomicInteger(0); - future = executor.submit(() -> { - while (true) { - if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { - sleep("Target HTTP server is down...", 3); - } - if (listOperations.size(redisKey) > 0) { - List list = listOperations.range(redisKey, -10, -1); - list.forEach(obj -> { - //TODO 2.5: Callback? - TbMsg msg = TbMsg.fromBytes((byte[]) obj, null); - log.debug("Trying to send the message: {}", msg); - listOperations.remove(redisKey, -1, obj); - httpClient.processMessage(ctx, msg, this); - }); - } else { - sleep("Queue is empty, waiting for tasks!", 1); - } - } - }); - } - - void destroy() { - if (future != null) { - future.cancel(true); - } - if (executor != null) { - executor.shutdownNow(); - } - } - - void push(TbMsg msg) { - listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); - if (trimQueue) { - listOperations.trim(redisKey, 0, validateMaxQueueSize()); - } - } - - void pushOnFailure(TbMsg msg) { - listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); - failuresCounter.incrementAndGet(); - } - - void resetCounter() { - failuresCounter.set(0); - } - - private String constructRedisKey() { - return ctx.getServiceId() + ctx.getSelfId(); - } - - private int validateMaxQueueSize() { - if (maxQueueSize != 0) { - return maxQueueSize; - } - return MAX_QUEUE_SIZE; - } - - private void sleep(String logMessage, int sleepSeconds) { - try { - log.debug(logMessage); - TimeUnit.SECONDS.sleep(sleepSeconds); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted!", e); - } - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9b9d275977..9cb171d0dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,8 +25,6 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.concurrent.ExecutionException; - @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -47,7 +45,6 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; - private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -55,20 +52,13 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - if (ctx.getRedisTemplate() == null) { - throw new RuntimeException("Redis cache type must be used!"); - } - queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); + log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - if (useRedisQueueForMsgPersistence) { - queueProcessor.push(msg); - } else { - httpClient.processMessage(ctx, msg, null); - } + public void onMsg(TbContext ctx, TbMsg msg) { + httpClient.processMessage(ctx, msg); } @Override @@ -76,9 +66,6 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } - if (this.queueProcessor != null) { - this.queueProcessor.destroy(); - } } } From e991c0ef23b96b7bc86967d108984391ef1fa925 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 08:44:34 +0300 Subject: [PATCH 049/102] No more Synchronization nodes --- .../server/actors/ActorSystemContext.java | 6 - .../actors/ruleChain/DefaultTbContext.java | 6 - .../BaseRuleChainTransactionService.java | 225 ------------------ .../transaction/TbTransactionTask.java | 44 ---- .../thingsboard/server/common/msg/TbMsg.java | 30 +-- .../common/msg/TbMsgTransactionData.java | 30 --- common/message/src/main/proto/tbmsg.proto | 9 +- .../api/RuleChainTransactionService.java | 28 --- .../rule/engine/api/TbContext.java | 2 - .../TbSynchronizationBeginNode.java | 24 +- .../transaction/TbSynchronizationEndNode.java | 11 +- 11 files changed, 13 insertions(+), 402 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java delete mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index fb4ff77359..7e9b68882b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -36,7 +36,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -67,7 +66,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -246,10 +244,6 @@ public class ActorSystemContext { @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; - @Autowired(required = false) - @Getter - private RuleChainTransactionService ruleChainTransactionService; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 13f7b1122d..b268236448 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -24,7 +24,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -378,11 +377,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } - @Override - public RuleChainTransactionService getRuleChainTransactionService() { - return mainCtx.getRuleChainTransactionService(); - } - @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java deleted file mode 100644 index 477f115a96..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -@Service -@TbRuleEngineComponent -@Slf4j -public class BaseRuleChainTransactionService implements RuleChainTransactionService { - - private final DbCallbackExecutorService callbackExecutor; - - @Value("${actors.rule.transaction.queue_size}") - private int finalQueueSize; - @Value("${actors.rule.transaction.duration}") - private long duration; - - private final Lock transactionLock = new ReentrantLock(); - private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); - private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); - - private ExecutorService timeoutExecutor; - - public BaseRuleChainTransactionService(DbCallbackExecutorService callbackExecutor) { - this.callbackExecutor = callbackExecutor; - } - - @PostConstruct - public void init() { - timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); - executeOnTimeout(); - } - - @PreDestroy - public void destroy() { - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - } - - @Override - public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); - int queueSize = queue.size(); - if (queueSize >= finalQueueSize) { - log.trace("Queue has no space: {}", transactionTask); - executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); - } else { - addMsgToQueues(queue, transactionTask); - if (queueSize == 0) { - executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); - } else { - log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); - } - } - } finally { - transactionLock.unlock(); - } - } - - @Override - public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - //TODO 2.5 -// Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); -// if (address.isPresent()) { -// sendTransactionEventToRemoteServer(msg, address.get()); -// executeOnSuccess(onSuccess, msg); -// } else { - endLocalTransaction(msg, onSuccess, onFailure); -// } - } - - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { - queue.offer(transactionTask); - timeoutQueue.offer(transactionTask); - log.trace("Added msg to queue, size: [{}]", queue.size()); - } - - private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask currentTransactionTask = queue.peek(); - if (currentTransactionTask != null) { - if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { - currentTransactionTask.setCompleted(true); - queue.poll(); - log.trace("Removed msg from queue, size [{}]", queue.size()); - - executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); - executeOnSuccess(onSuccess, msg); - - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } else { - log.trace("Task has expired!"); - executeOnFailure(onFailure, "Task has expired!"); - } - } else { - log.trace("Queue is empty, previous task has expired!"); - executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); - } - } finally { - transactionLock.unlock(); - } - } - - private void executeOnTimeout() { - timeoutExecutor.submit(() -> { - while (true) { - TbTransactionTask transactionTask = timeoutQueue.peek(); - if (transactionTask != null) { - long sleepDuration = 0L; - transactionLock.lock(); - try { - if (transactionTask.isCompleted()) { - timeoutQueue.poll(); - } else { - long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); - if (expIn < 0) { - log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - timeoutQueue.poll(); - executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); - - BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); - if (queue != null) { - queue.poll(); - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } - } else { - sleepDuration = Math.min(expIn, duration); - } - } - } finally { - transactionLock.unlock(); - } - if (sleepDuration > 0L) { - try { - log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - TimeUnit.MILLISECONDS.sleep(sleepDuration); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } else { - try { - log.trace("Queue is empty, waiting for tasks!"); - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } - }); - } - - private void executeOnFailure(Consumer onFailure, String exception) { - executeCallback(() -> { - onFailure.accept(new RuntimeException(exception)); - return null; - }); - } - - private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { - executeCallback(() -> { - onSuccess.accept(tbMsg); - return null; - }); - } - - private void executeCallback(Callable task) { - callbackExecutor.executeAsync(task); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java deleted file mode 100644 index c86f882ff7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -@Data -@AllArgsConstructor -public final class TbTransactionTask { - - private final TbMsg msg; - private final Consumer onStart; - private final Consumer onEnd; - private final Consumer onFailure; - private final long expirationTime; - - private boolean isCompleted; - - public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { - this.msg = msg; - this.onStart = onStart; - this.onEnd = onEnd; - this.onFailure = onFailure; - this.expirationTime = expirationTime; - this.isCompleted = false; - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 84110ed163..2e518df084 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -44,7 +44,6 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; - private final TbMsgTransactionData transactionData; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; //This field is not serialized because we use queues and there is no need to do it @@ -68,27 +67,22 @@ public final class TbMsg implements Serializable { public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); } public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); } private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); - } - - private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; this.dataType = dataType; this.data = data; - this.transactionData = transactionData; this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; if (callback != null) { @@ -125,15 +119,6 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } - TbMsgTransactionData transactionData = msg.getTransactionData(); - if (transactionData != null) { - MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); - transactionBuilder.setId(transactionData.getTransactionId().toString()); - transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); - transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); - transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); - builder.setTransactionData(transactionBuilder.build()); - } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); @@ -144,9 +129,6 @@ public final class TbMsg implements Serializable { try { MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); - EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), - new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); - TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; @@ -157,17 +139,17 @@ public final class TbMsg implements Serializable { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, callback); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, this.transactionData, ruleChainId, null, callback); + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); } public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, this.transactionData, ruleChainId, ruleNodeId, callback); + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java deleted file mode 100644 index 5d57514275..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.io.Serializable; -import java.util.UUID; - -@Data -public final class TbMsgTransactionData implements Serializable { - - private final UUID transactionId; - private final EntityId originatorId; - -} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 03dc17b5c9..737d5f6f57 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,13 +23,6 @@ message TbMsgMetaDataProto { map data = 1; } -message TbMsgTransactionDataProto { - string id = 1; - string entityType = 2; - int64 entityIdMSB = 3; - int64 entityIdLSB = 4; -} - message TbMsgProto { string id = 1; string type = 2; @@ -46,7 +39,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - TbMsgTransactionDataProto transactionData = 12; + //Transaction Data (12) was removed in 2.5 int32 dataType = 13; string data = 14; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java deleted file mode 100644 index 1bb56adb1d..0000000000 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.api; - -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -public interface RuleChainTransactionService { - - void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - - void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - -} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index daab510102..9b1edc5f5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -203,8 +203,6 @@ public interface TbContext { String getServiceId(); - RuleChainTransactionService getRuleChainTransactionService(); - EventLoopGroup getSharedEventLoop(); CassandraCluster getCassandraCluster(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index b8eff0901a..31501688eb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,10 +25,6 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgTransactionData; - -import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -43,31 +39,17 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") +@Deprecated public class TbSynchronizationBeginNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); - - TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); - //TODO 2.5: Callback? - TbMsg tbMsg = TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - - ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { - log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); - ctx.tellNext(startMsg, SUCCESS); - }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), - throwable -> { - log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); - ctx.tellFailure(tbMsg, throwable); - }); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index af3742a0fe..ac3d42ef36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -40,25 +40,20 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) +@Deprecated public class TbSynchronizationEndNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - ctx.getRuleChainTransactionService().endTransaction(msg, - successMsg -> ctx.tellNext(successMsg, SUCCESS), - throwable -> ctx.tellFailure(msg, throwable)); - log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override public void destroy() { - } } From 69c42d243bfc62baef2cedf5f1b3bc81c8fda97e Mon Sep 17 00:00:00 2001 From: nordmif Date: Tue, 14 Apr 2020 14:15:57 +0300 Subject: [PATCH 050/102] changed device attributes name to clarify its functionality --- .../thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 653b571b4b..03f0652c3f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "device attributes", + name = "related device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + From 0c54f836b33a5720635b7b1216424d7a14fd7c42 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 16:17:08 +0300 Subject: [PATCH 051/102] Fixed tests --- .../server/actors/app/AppActor.java | 29 +++++++++++---- .../server/actors/tenant/TenantActor.java | 17 ++++++--- .../server/controller/BaseController.java | 9 ++++- .../state/DefaultDeviceStateService.java | 2 ++ .../DefaultRuleEngineStatisticsService.java | 21 +++++++---- application/src/main/resources/logback.xml | 1 - .../src/main/resources/thingsboard.yml | 8 ++--- .../controller/AbstractControllerTest.java | 2 ++ .../BaseEntityViewControllerTest.java | 18 ++++++++-- .../AbstractMqttTelemetryIntegrationTest.java | 17 +++++---- ...AbstractRuleEngineFlowIntegrationTest.java | 36 +++++++++---------- ...actRuleEngineLifecycleIntegrationTest.java | 23 ++++++------ application/src/test/resources/logback.xml | 2 +- .../thingsboard/server/common/msg/TbMsg.java | 4 +++ .../common/DefaultTbQueueRequestTemplate.java | 3 +- .../server/queue/kafka/TBKafkaAdmin.java | 25 +------------ .../server/queue/memory/InMemoryStorage.java | 9 ++--- .../settings/TbQueueRuleEngineSettings.java | 5 +-- common/queue/src/main/proto/queue.proto | 14 ++++---- .../test/resources/cassandra-test.properties | 1 + dao/src/test/resources/sql-test.properties | 2 ++ .../rule/engine/action/TbAlarmNodeTest.java | 12 ++++++- .../TbGetCustomerAttributeNodeTest.java | 4 +-- .../transform/TbTransformMsgNodeTest.java | 2 +- 24 files changed, 159 insertions(+), 107 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 7e4773aa8b..b7782a6a87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -42,21 +42,26 @@ import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; + private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); + this.deletedTenants = new HashSet<>(); } @Override @@ -139,7 +144,11 @@ public class AppActor extends ContextAwareActor { if (SYSTEM_TENANT.equals(msg.getTenantId())) { msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + } else { + msg.getTbMsg().getCallback().onSuccess(); + } } } @@ -154,8 +163,10 @@ public class AppActor extends ContextAwareActor { } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + TenantId tenantId = new TenantId(msg.getEntityId().getId()); + deletedTenants.add(tenantId); + ActorRef tenantActor = tenantActors.get(tenantId); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -172,16 +183,22 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + } else { + if (msg instanceof TransportToDeviceActorMsgWrapper) { + ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); + } + } } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.debug("[{}] Creating tenant actor.", tenantId); + log.info("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index a1bf343efe..894c573e15 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -66,6 +66,8 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } + boolean cantFindTenant = false; + @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); @@ -78,10 +80,14 @@ public class TenantActor extends RuleChainManagerActor { isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); if (isRuleEngineForCurrentTenant) { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; } } log.info("[{}] Tenant actor started.", tenantId); @@ -97,6 +103,9 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { + if (cantFindTenant) { + log.info("Missing Tenant msg: {}", msg); + } switch (msg.getMsgType()) { case PARTITION_CHANGE_MSG: PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 73b25d4de2..8385e61a20 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; @@ -667,7 +668,13 @@ public abstract class BaseController { } } TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - tbClusterService.pushMsgToRuleEngine(user.getTenantId(), entityId, tbMsg, null); + TenantId tenantId = user.getTenantId(); + if (tenantId.isNullUid()) { + if (entity instanceof HasTenantId) { + tenantId = ((HasTenantId) entity).getTenantId(); + } + } + tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 1e3ab73a26..85a4ac2669 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -325,6 +325,8 @@ public class DefaultDeviceStateService implements DeviceStateService { }); }); + addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 38f2fe367a..3951703351 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -76,13 +77,19 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS String queueName = ruleEngineStats.getQueueName(); ruleEngineStats.getTenantStats().forEach((id, stats) -> { TenantId tenantId = new TenantId(id); - AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); - if (stats.getTotalMsgCounter().get() > 0) { - List tsList = stats.getCounters().entrySet().stream() - .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) - .collect(Collectors.toList()); - if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + try { + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + } catch (DataValidationException e) { + if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e; } } }); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 08a976592d..614f85ba0e 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,7 +27,6 @@ - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b5e875e07b..c138c23fc9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -412,7 +412,7 @@ audit-log: state: defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" - persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:true}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: evaluator: "${JS_EVALUATOR:local}" # local/remote @@ -513,7 +513,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -588,7 +588,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - - name: "Main" + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 3e9f8c7853..26f375c464 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -197,6 +198,7 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); + log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 1eb821f06a..e60f549610 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -24,6 +25,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -46,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -55,6 +58,7 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; +@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -417,12 +421,22 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); - + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); +// client.disconnect(); + } + + private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { + long start = System.currentTimeMillis(); + while (!client.isConnected()) { + Thread.sleep(100); + if (start + ms < System.currentTimeMillis()) { + throw new RuntimeException("Client is not connected!"); + } + } } private Set getTelemetryKeys(String type, String id) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f1580c65c7..122bef2e26 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -80,7 +80,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); Thread.sleep(1000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -104,13 +104,17 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); - client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); + client.connect(options).waitForCompletion(3000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; - String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); +// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. +// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) +// MqttClient <- SUB_ACK <- Transport + Thread.sleep(1000); + doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -120,8 +124,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private Integer qoS; - private String payload; + private volatile Integer qoS; + private volatile String payload; String getPayload() { return payload; @@ -138,6 +142,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { + log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 7fe63827eb..3343d271fe 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.rules.flow; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; @@ -35,6 +38,8 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; @@ -55,7 +60,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -142,15 +147,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), TbMsgDataType.JSON, "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -257,17 +259,13 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - TbMsgDataType.JSON, - "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 78129917ef..17f17698c9 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.rules.lifecycle; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -39,6 +42,8 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; @@ -58,7 +63,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -133,17 +138,13 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - TbMsgDataType.JSON, - "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..64eac9e61d 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 2e518df084..3edf4e5061 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -65,6 +65,10 @@ public final class TbMsg implements Serializable { return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); } + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + } + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e8ebb934d3..f02ae63441 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -95,14 +95,13 @@ public class DefaultTbQueueRequestTemplate { - log.trace("Received response to Queue Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; if (requestIdHeader == null) { log.error("[{}] Missing requestId in header and body", response); } else { requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received", requestId); + log.trace("[{}] Response received: {}", requestId, response); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index e1ca580c28..76c977e25a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -41,7 +41,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } - //TODO 2.5 + //TODO 2.5 - ybondarenko Need to pass not only settings but also properties for topic creation. Somewhere in thingsboard.yml, in KV format. @Override public void createTopicIfNotExists(String topic) { try { @@ -57,29 +57,6 @@ public class TBKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } -// -// KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); -// -// ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); -// -// return Futures.transformAsync(topicFuture, topicDescription -> { -// KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); -// return JdkFutureAdapters.listenInPoolThread(resultFuture); -// }); - } - - public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { - synchronized (this) { - long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); - while (!topicExists(topic)) { - long waitMs = timeoutExpiredMs - System.currentTimeMillis(); - if (waitMs <= 0) { - throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); - } else { - wait(1000); - } - } - } } public CreateTopicsResult createTopic(NewTopic topic) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index c83fa02fab..f919df871e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -60,16 +60,13 @@ public final class InMemoryStorage { if (first != null) { entities = new ArrayList<>(); entities.add(first); - } else { - entities = Collections.emptyList(); List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 100); + storage.get(topic).drainTo(otherList, 999); for (TbQueueMsg other : otherList) { entities.add((T) other); } - } - if (entities.size() > 0) { - storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).addAll(entities); + } else { + entities = Collections.emptyList(); } return entities; } catch (InterruptedException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index 95540798fc..9dfe715ece 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -36,11 +36,12 @@ public class TbQueueRuleEngineSettings { private String topic; private List queues; - //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. See how ther are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. + // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers @PostConstruct public void validate() { queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { - log.warn("Main queue is not configured in thingsboard.yml"); + log.error("Main queue is not configured in thingsboard.yml"); return new RuntimeException("No \"Main\" queue configured!"); }); } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index dac64a3478..51bc7649be 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -400,11 +400,11 @@ message ToRuleEngineNotificationMsg { /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index af9a4b356f..51f34a08d6 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,3 +60,4 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false +service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 765e0da3d6..13c0fbc818 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,6 +16,8 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 +service.type=monolith + #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 2fcdc6c838..61b83d647d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -47,6 +48,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; +import java.util.function.Consumer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -82,6 +84,11 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; + @Captor + private ArgumentCaptor successCaptor; + @Captor + private ArgumentCaptor> failureCaptor; + private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -119,11 +126,12 @@ public class TbAlarmNodeTest { when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); - doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -191,6 +199,8 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 211459b84c..a0a61ae30f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -256,7 +256,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -268,7 +268,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 3d40f3acd0..a00e97cf1c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -77,7 +77,7 @@ public class TbTransformMsgNodeTest { node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); + verify(ctx).tellSuccess(captor.capture()); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } From e1f914e2fad9917e1a60c6681ac5837a6675495c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 16:47:01 +0300 Subject: [PATCH 052/102] Restored log level --- application/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 64eac9e61d..47dacce343 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + From e8b94d566cff1f7c85d6707370a535041f420da2 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 15 Apr 2020 16:57:40 +0300 Subject: [PATCH 053/102] init commit --- ...rvice.java => AbstractCleanUpService.java} | 40 ++++--------- .../ttl/events/EventsCleanUpService.java | 57 +++++++++++++++++++ .../AbstractTimeseriesCleanUpService.java | 49 ++++++++++++++++ .../PsqlTimeseriesCleanUpService.java | 2 +- .../TimescaleTimeseriesCleanUpService.java | 2 +- .../src/main/resources/thingsboard.yml | 11 +++- 6 files changed, 127 insertions(+), 34 deletions(-) rename application/src/main/java/org/thingsboard/server/service/ttl/{AbstractTimeseriesCleanUpService.java => AbstractCleanUpService.java} (67%) create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/PsqlTimeseriesCleanUpService.java (96%) rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/TimescaleTimeseriesCleanUpService.java (95%) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 2399450d86..61d81ae0b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlDao; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService { - - @Value("${sql.ttl.ts_key_value_ttl}") - protected long systemTtl; - @Value("${sql.ttl.enabled}") - private boolean ttlTaskExecutionEnabled; +@Slf4j +@PsqlDao +public abstract class AbstractCleanUpService { @Value("${spring.datasource.url}") - private String dbUrl; + protected String dbUrl; @Value("${spring.datasource.username}") - private String dbUserName; + protected String dbUserName; @Value("${spring.datasource.password}") - private String dbPassword; - - @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - protected abstract void doCleanUp(Connection connection); + protected String dbPassword; protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService { return removed; } - private void getWarnings(Statement statement) throws SQLException { + protected void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService { } } -} \ No newline at end of file + protected abstract void doCleanUp(Connection connection); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java new file mode 100644 index 0000000000..bb7d112bb1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.events; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlDao +@Slf4j +@Service +public class EventsCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.events.events_key_value_ttl}") + private long ttl; + + @Value("${sql.ttl.events.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + @Override + protected void doCleanUp(Connection connection) { + log.info("ttl: [{}]", ttl); + log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); + // TODO: 4/15/20 Do a clean up. + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java new file mode 100644 index 0000000000..75b07b9176 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlTsAnyDao +@Slf4j +public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.ts.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.ts.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index ab344dc518..cd403ee3b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index 1dbdb4ad55..f5898b9b20 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0402c91387..9283f9283c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -217,9 +217,14 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - enabled: "${SQL_TTL_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds - ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds + ts: + enabled: "${SQL_TTL_TS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds + events: + enabled: "${SQL_TTL_EVENTS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: From d654e09d6a52c0a0d038ac176f36214c0dce25b2 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Wed, 15 Apr 2020 17:47:46 +0300 Subject: [PATCH 054/102] kafka topic settings * added topic-properties to kafka queue * added topic-properties to kafka queue to transport * kafka topic settings improvements --- .../src/main/resources/thingsboard.yml | 7 ++ .../server/queue/kafka/TBKafkaAdmin.java | 43 +++++++----- .../queue/kafka/TBKafkaConsumerTemplate.java | 8 ++- .../queue/kafka/TBKafkaProducerTemplate.java | 26 ++++++- .../server/queue/kafka/TbKafkaSettings.java | 8 ++- .../queue/kafka/TbKafkaTopicConfigs.java | 69 +++++++++++++++++++ .../provider/KafkaMonolithQueueFactory.java | 34 +++++++-- .../provider/KafkaTbCoreQueueFactory.java | 32 +++++++-- .../KafkaTbRuleEngineQueueFactory.java | 28 ++++++-- .../KafkaTbTransportQueueFactory.java | 42 +++++++---- .../src/main/resources/tb-coap-transport.yml | 7 ++ .../src/main/resources/tb-http-transport.yml | 7 ++ .../src/main/resources/tb-mqtt-transport.yml | 7 ++ 13 files changed, 267 insertions(+), 51 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c138c23fc9..7db2224d34 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -521,6 +521,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index 76c977e25a..69ad6286a6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -19,15 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbQueueAdmin; import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. @@ -35,17 +34,34 @@ import java.util.concurrent.TimeoutException; @Slf4j public class TBKafkaAdmin implements TbQueueAdmin { - AdminClient client; + private final AdminClient client; + private final Map topicConfigs; + private final Set topics = ConcurrentHashMap.newKeySet(); - public TBKafkaAdmin(TbKafkaSettings settings) { + private final short replicationFactor; + + public TBKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { client = AdminClient.create(settings.toProps()); + this.topicConfigs = topicConfigs; + + try { + topics.addAll(client.listTopics().names().get()); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to get all topics.", e); + } + + replicationFactor = settings.getReplicationFactor(); } - //TODO 2.5 - ybondarenko Need to pass not only settings but also properties for topic creation. Somewhere in thingsboard.yml, in KV format. @Override public void createTopicIfNotExists(String topic) { + if (topics.contains(topic)) { + return; + } try { - createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic).get(); + NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + createTopic(newTopic).values().get(topic).get(); + topics.add(topic); } catch (ExecutionException ee) { if (ee.getCause() instanceof TopicExistsException) { //do nothing @@ -57,19 +73,10 @@ public class TBKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } + } public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } - - private boolean topicExists(String topic) throws InterruptedException { - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - try { - topicDescriptionFuture.get(); - return true; - } catch (ExecutionException e) { - return false; - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index d882153177..069f4a6455 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -23,6 +23,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; @@ -43,7 +44,7 @@ import java.util.stream.Collectors; @Slf4j public class TBKafkaConsumerTemplate implements TbQueueConsumer { - private final TBKafkaAdmin admin; + private final TbQueueAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; @@ -57,7 +58,8 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords) { + int maxPollRecords, + TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); if (groupId != null) { @@ -70,7 +72,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon if (maxPollRecords > 0) { props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); } - this.admin = new TBKafkaAdmin(settings); + this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 2d82ed275d..f81381bc38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -24,12 +24,15 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** @@ -46,8 +49,12 @@ public class TBKafkaProducerTemplate implements TbQueuePro @Getter private final TbKafkaSettings settings; + private final TbQueueAdmin admin; + + private final Set topics; + @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId, TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); @@ -57,6 +64,8 @@ public class TBKafkaProducerTemplate implements TbQueuePro this.settings = settings; this.producer = new KafkaProducer<>(props); this.defaultTopic = defaultTopic; + this.admin = admin; + topics = ConcurrentHashMap.newKeySet(); } @Override @@ -65,6 +74,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + createTopicIfNotExist(tpi); String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; @@ -85,8 +95,18 @@ public class TBKafkaProducerTemplate implements TbQueuePro }); } + private void createTopicIfNotExist(TopicPartitionInfo tpi) { + if (topics.contains(tpi)) { + return; + } + admin.createTopicIfNotExists(tpi.getFullTopicName()); + topics.add(tpi); + } + @Override public void stop() { - + if (producer != null) { + producer.close(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index e8581d9719..66121cb215 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.kafka; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; @@ -32,9 +33,6 @@ import java.util.Properties; @Component public class TbKafkaSettings { - static final String REQUEST_ID_HEADER = "requestId"; - static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - @Value("${queue.kafka.bootstrap.servers}") private String servers; @@ -53,6 +51,10 @@ public class TbKafkaSettings { @Value("${queue.kafka.buffer.memory}") private long bufferMemory; + @Value("${queue.kafka.replication_factor}") + @Getter + private short replicationFactor; + @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java new file mode 100644 index 0000000000..8682ca385f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +public class TbKafkaTopicConfigs { + @Value("${queue.kafka.topic-properties.core}") + private String coreProperties; + @Value("${queue.kafka.topic-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.kafka.topic-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.kafka.topic-properties.notifications}") + private String notificationsProperties; + @Value("${queue.kafka.topic-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dae4221e91..9eb656c2ea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -62,13 +64,20 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -77,6 +86,12 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -85,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); + requestBuilder.admin(notificationAdmin); return requestBuilder.build(); } @@ -94,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -103,6 +120,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -112,6 +130,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -121,6 +140,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -133,6 +153,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); return consumerBuilder.build(); } @@ -144,6 +165,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -155,6 +177,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); return consumerBuilder.build(); } @@ -166,6 +189,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -177,6 +201,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); return consumerBuilder.build(); } @@ -186,6 +211,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); + requestBuilder.admin(transportApiAdmin); return requestBuilder.build(); } @@ -196,24 +222,24 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index e148416bb4..8800c1c55a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -59,12 +61,19 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -72,6 +81,12 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -80,6 +95,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -89,6 +105,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -98,6 +115,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -107,6 +125,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -116,6 +135,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -127,6 +147,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); return consumerBuilder.build(); } @@ -138,6 +159,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -149,6 +171,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); return consumerBuilder.build(); } @@ -158,6 +181,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -168,24 +192,24 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 8262d27354..b96d4ebe86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -38,6 +39,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -56,17 +58,28 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -75,6 +88,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -84,6 +98,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -93,6 +108,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -103,6 +119,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -112,6 +129,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -124,6 +142,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); return consumerBuilder.build(); } @@ -135,6 +154,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -145,24 +165,24 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 12f677af9d..f5a31cf1c7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -18,25 +18,27 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -50,18 +52,29 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -70,6 +83,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.admin(transportApiAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); @@ -77,10 +91,11 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(transportApiAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(requestBuilder.build()); templateBuilder.responseTemplate(responseBuilder.build()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -93,8 +108,9 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); + requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -104,6 +120,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -115,6 +132,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(notificationAdmin); return responseBuilder.build(); } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index b94148f608..e2fb7e149a 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -50,6 +50,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0fa2788cac..dfd2c14be2 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -51,6 +51,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 747427caf8..81f0fb795d 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -81,6 +81,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" From 0a136588eb913f1b699132cb454d1ec6ea76c4f8 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 15 Apr 2020 17:48:47 +0300 Subject: [PATCH 055/102] CheckPoint Node ui, Queue Controller, queue-type-list directive (#2612) * init commit * ui: service for queue * merge with develop/2.5 & fix license * queue directive * queue directive * queue-type-list directive & TbCheckPointNode * added rulenode-core-config.js file * added new rulenode-core-config.js * Develop/2.5 check point node (#3) * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * Develop/2.5 check point node (#4) * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 Co-authored-by: Dmitriymush Co-authored-by: Dmitriy Mushat <54553744+Dmitriymush@users.noreply.github.com> --- .../server/controller/QueueController.java | 54 +++++++++ .../rule/engine/flow/TbAckNode.java | 10 +- .../rule/engine/flow/TbCheckpointNode.java | 6 +- .../static/rulenode/rulenode-core-config.js | 8 +- ui/src/app/api/queue.service.js | 40 +++++++ ui/src/app/components/queue/index.js | 23 ++++ .../queue/queue-type-list.directive.js | 111 ++++++++++++++++++ .../app/components/queue/queue-type-list.scss | 15 +++ .../components/queue/queue-type-list.tpl.html | 43 +++++++ ui/src/app/layout/index.js | 2 + ui/src/app/locale/locale.constant-en_US.json | 6 + ui/src/app/locale/locale.constant-ru_RU.json | 6 + ui/src/app/locale/locale.constant-uk_UA.json | 6 + 13 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/QueueController.java create mode 100644 ui/src/app/api/queue.service.js create mode 100644 ui/src/app/components/queue/index.js create mode 100644 ui/src/app/components/queue/queue-type-list.directive.js create mode 100644 ui/src/app/components/queue/queue-type-list.scss create mode 100644 ui/src/app/components/queue/queue-type-list.tpl.html diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java new file mode 100644 index 0000000000..8d1cec7f4b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +public class QueueController extends BaseController { + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) + @ResponseBody + public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { + checkParameter("serviceType", serviceType); + try { + ServiceType type = ServiceType.valueOf(serviceType); + switch (type) { + case TB_RULE_ENGINE: + return Arrays.asList("HighPriority", "Main"); + default: + return Collections.emptyList(); + } + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java index 9351e970ed..b5af3f1563 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -22,7 +22,6 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -33,12 +32,17 @@ import org.thingsboard.server.common.msg.TbMsg; name = "acknowledge", configClazz = EmptyNodeConfiguration.class, nodeDescription = "Acknowledges the incoming message", - nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.") - + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbNodeEmptyConfig" +) public class TbAckNode implements TbNode { + EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java index 0364251076..d5048eadbd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -35,8 +35,10 @@ import static org.thingsboard.common.util.DonAsynchron.withCallback; name = "checkpoint", configClazz = TbCheckpointNodeConfiguration.class, nodeDescription = "transfers the message to another queue", - nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.") - + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbActionNodeCheckPointConfig" +) public class TbCheckpointNode implements TbNode { private TbCheckpointNodeConfiguration config; diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index fd5d9b7c62..f848a91875 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -1,6 +1,6 @@ -!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(103)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; +!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(105)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
tb.rulenode.select-queue-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; },function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; -},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},31,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(75),r=a(i),o=n(53),l=a(o),s=n(58),d=a(s),u=n(55),c=a(u),m=n(54),g=a(m),p=n(62),f=a(p),b=n(69),v=a(b),y=n(70),h=a(y),q=n(68),k=a(q),x=n(61),$=a(x),T=n(73),C=a(T),w=n(74),M=a(w),S=n(67),N=a(S),_=n(63),F=a(_),E=n(72),P=a(E),A=n(65),V=a(A),I=n(64),O=a(I),j=n(52),D=a(j),L=n(76),R=a(L),K=n(57),U=a(K),z=n(56),H=a(z),B=n(71),G=a(B),Y=n(59),Q=a(Y),W=n(66),J=a(W);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",N.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",O.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader; -t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(84),r=a(i),o=n(85),l=a(o),s=n(80),d=a(s),u=n(86),c=a(u),m=n(79),g=a(m),p=n(87),f=a(p),b=n(82),v=a(b),y=n(81),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(95),r=a(i),o=n(93),l=a(o),s=n(96),d=a(s),u=n(90),c=a(u),m=n(94),g=a(m),p=n(89),f=a(p),b=n(91),v=a(b),y=n(88),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(47),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(99),r=a(i),o=n(101),l=a(o),s=n(102),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(106),r=a(i),o=n(92),l=a(o),s=n(83),d=a(s),u=n(100),c=a(u),m=n(60),g=a(m),p=n(78),f=a(p),b=n(98),v=a(b),y=n(77),h=a(y),q=n(97),k=a(q),x=n(105),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata", -header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16"},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(104),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); +},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},32,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.serviceType="TB_RULE_ENGINE",n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(77),r=a(i),o=n(54),l=a(o),s=n(60),d=a(s),u=n(57),c=a(u),m=n(56),g=a(m),p=n(64),f=a(p),b=n(71),v=a(b),y=n(72),h=a(y),q=n(70),k=a(q),x=n(63),$=a(x),T=n(75),C=a(T),w=n(76),M=a(w),N=n(69),S=a(N),_=n(65),F=a(_),E=n(74),P=a(E),A=n(67),V=a(A),I=n(66),j=a(I),O=n(53),D=a(O),L=n(78),R=a(L),K=n(59),U=a(K),z=n(58),H=a(z),B=n(73),G=a(B),Y=n(61),Q=a(Y),W=n(68),J=a(W),Z=n(55),X=a(Z);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",S.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",j.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).directive("tbActionNodeCheckPointConfig",X.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){ +var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader;t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(86),r=a(i),o=n(87),l=a(o),s=n(82),d=a(s),u=n(88),c=a(u),m=n(81),g=a(m),p=n(89),f=a(p),b=n(84),v=a(b),y=n(83),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(43),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(97),r=a(i),o=n(95),l=a(o),s=n(98),d=a(s),u=n(92),c=a(u),m=n(96),g=a(m),p=n(91),f=a(p),b=n(93),v=a(b),y=n(90),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(101),r=a(i),o=n(103),l=a(o),s=n(104),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(52),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(108),r=a(i),o=n(94),l=a(o),s=n(85),d=a(s),u=n(102),c=a(u),m=n(62),g=a(m),p=n(80),f=a(p),b=n(100),v=a(b),y=n(79),h=a(y),q=n(99),k=a(q),x=n(107),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required", +"endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(106),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); //# sourceMappingURL=rulenode-core-config.js.map \ No newline at end of file diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js new file mode 100644 index 0000000000..0fe93e5fa6 --- /dev/null +++ b/ui/src/app/api/queue.service.js @@ -0,0 +1,40 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default angular.module('thingsboard.api.queue', []) + .factory('queueService', queueService) + .name; + +/*@ngInject*/ +function queueService($http, $q) { + var service = { + getTenantQueuesByServiceType: getTenantQueuesByServiceType + }; + + return service; + + function getTenantQueuesByServiceType(serviceType, config) { + let deferred = $q.defer(); + let url = '/api/tenant/queues?serviceType=' + serviceType; + + $http.get(url, config).then(function success(data) { + deferred.resolve(data); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } +} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js new file mode 100644 index 0000000000..5236603270 --- /dev/null +++ b/ui/src/app/components/queue/index.js @@ -0,0 +1,23 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import thingsboardApiQueue from '../../api/queue.service'; +import queueTypeList from "./queue-type-list.directive"; + +export default angular.module('thingsboard.queue', [ + thingsboardApiQueue +]) + .directive('tbQueueTypeList', queueTypeList) + .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js new file mode 100644 index 0000000000..6a2f99d280 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.directive.js @@ -0,0 +1,111 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import './queue-type-list.scss'; + +/* eslint-disable import/no-unresolved, import/default */ + +import queueTypeListTemplate from './queue-type-list.tpl.html'; + +/* eslint-enable import/no-unresolved, import/default */ + +/*@ngInject*/ +export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { + + var linker = function (scope, element, attrs, ngModelCtrl) { + var template = $templateCache.get(queueTypeListTemplate); + element.html(template); + + scope.queues = null; + scope.queue = null; + scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; + scope.queueSearchText = ''; + + scope.fetchQueues = function(searchText) { + var deferred = $q.defer(); + loadQueues().then( + function success(queueArr) { + let result = $filter('filter')(queueArr, {'$': searchText}); + if (result && result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + deferred.resolve(result); + } else { + deferred.resolve([searchText]); + } + }, + function fail() { + deferred.reject(); + } + ); + + return deferred.promise; + }; + + scope.updateView = function () { + if (!scope.disabled) { + ngModelCtrl.$setViewValue(scope.queue); + } + }; + + function loadQueues() { + var deferred = $q.defer(); + if (!scope.queues) { + queueService.getTenantQueuesByServiceType(scope.queueType).then( + function success(queueArr) { + scope.queues = queueArr.data; + deferred.resolve(scope.queues); + }, + function fail() { + deferred.reject(); + } + ); + } else { + deferred.resolve(scope.queues); + } + return deferred.promise; + } + + ngModelCtrl.$render = function () { + scope.queue = ngModelCtrl.$viewValue; + }; + + scope.$watch('queue', function (newValue, prevValue) { + if (!angular.equals(newValue, prevValue)) { + scope.updateView(); + } + }); + + scope.$watch('disabled', function () { + scope.updateView(); + }); + + $compile(element.contents())(scope); + }; + + return { + restrict: "E", + require: "^ngModel", + link: linker, + scope: { + theForm: '=?', + tbRequired: '=?', + disabled:'=ngDisabled', + queueType: '=?' + } + }; +} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss new file mode 100644 index 0000000000..b8f6f58fb2 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.scss @@ -0,0 +1,15 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html new file mode 100644 index 0000000000..5fc700026e --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.tpl.html @@ -0,0 +1,43 @@ + + + +
+ {{item}} +
+ +
+
{{'queue.name_required' | translate}}
+
+ \ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index a00baa5199..f3b4d10e4f 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,6 +55,7 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; +import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -86,6 +87,7 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, + thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index e68ccec157..606fabd375 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1484,6 +1484,12 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index f86eb988ff..e93c92124c 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,6 +1403,12 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index 5a0899014d..c19049bc11 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,6 +1820,12 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", From 9fa82078a02d56ce1f3835bc6788b97ed52c54c8 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 12:43:10 +0300 Subject: [PATCH 056/102] created TbAwsSqsQueueAttributes and added queue properties to yml --- .../src/main/resources/thingsboard.yml | 8 +- .../server/queue/TbQueueAdmin.java | 1 + .../azure/servicebus/TbServiceBusAdmin.java | 4 +- .../{TBKafkaAdmin.java => TbKafkaAdmin.java} | 11 ++- ...late.java => TbKafkaConsumerTemplate.java} | 4 +- .../server/queue/kafka/TbKafkaHandler.java | 27 ------ .../queue/kafka/TbKafkaPartitioner.java | 30 ------- ...late.java => TbKafkaProducerTemplate.java} | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 2 + .../provider/AwsSqsMonolithQueueFactory.java | 40 +++++---- .../provider/AwsSqsTbCoreQueueFactory.java | 36 +++++--- .../AwsSqsTbRuleEngineQueueFactory.java | 30 ++++--- .../provider/AwsSqsTransportQueueFactory.java | 24 ++++-- .../InMemoryTbTransportQueueFactory.java | 11 ++- .../provider/KafkaMonolithQueueFactory.java | 42 +++++----- .../provider/KafkaTbCoreQueueFactory.java | 38 ++++----- .../KafkaTbRuleEngineQueueFactory.java | 32 +++---- .../KafkaTbTransportQueueFactory.java | 24 +++--- .../server/queue/pubsub/TbPubSubAdmin.java | 5 ++ .../queue/rabbitmq/TbRabbitMqAdmin.java | 7 +- .../server/queue/sqs/TbAwsSqsAdmin.java | 59 +++++++------ .../queue/sqs/TbAwsSqsQueueAttributes.java | 83 +++++++++++++++++++ .../server/queue/sqs/TbAwsSqsSettings.java | 2 - .../src/main/resources/tb-coap-transport.yml | 8 +- .../src/main/resources/tb-http-transport.yml | 8 +- .../src/main/resources/tb-mqtt-transport.yml | 8 +- 26 files changed, 334 insertions(+), 214 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaAdmin.java => TbKafkaAdmin.java} (92%) rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaConsumerTemplate.java => TbKafkaConsumerTemplate.java} (97%) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaProducerTemplate.java => TbKafkaProducerTemplate.java} (94%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7db2224d34..e1296275cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -533,7 +533,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index deb84f440e..bb541fb626 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -19,4 +19,5 @@ public interface TbQueueAdmin { void createTopicIfNotExists(String topic); + void destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index fb9cd89a8d..336514b7b2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -23,7 +23,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -68,8 +67,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } } - @PreDestroy - private void destroy() { + public void destroy() { try { client.close(); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java similarity index 92% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 69ad6286a6..b92b094af1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -32,7 +32,7 @@ import java.util.concurrent.ExecutionException; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaAdmin implements TbQueueAdmin { +public class TbKafkaAdmin implements TbQueueAdmin { private final AdminClient client; private final Map topicConfigs; @@ -40,7 +40,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { private final short replicationFactor; - public TBKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { + public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { client = AdminClient.create(settings.toProps()); this.topicConfigs = topicConfigs; @@ -76,6 +76,13 @@ public class TBKafkaAdmin implements TbQueueAdmin { } + @Override + public void destroy() { + if (client != null) { + client.close(); + } + } + public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 069f4a6455..49d1d74102 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -42,7 +42,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaConsumerTemplate implements TbQueueConsumer { +public class TbKafkaConsumerTemplate implements TbQueueConsumer { private final TbQueueAdmin admin; private final KafkaConsumer consumer; @@ -55,7 +55,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final String topic; @Builder - private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords, diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java deleted file mode 100644 index 8c2cbd2771..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import com.google.common.util.concurrent.ListenableFuture; - -/** - * Created by ashvayka on 05.10.18. - */ -public interface TbKafkaHandler { - - ListenableFuture handle(Request request); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java deleted file mode 100644 index b31f87642b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import org.apache.kafka.clients.producer.Partitioner; -import org.apache.kafka.common.PartitionInfo; - -import java.util.List; - -/** - * Created by ashvayka on 25.09.18. - */ -public interface TbKafkaPartitioner extends Partitioner { - - int partition(String topic, String key, T value, byte[] encodedValue, List partitions); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index f81381bc38..4f26f51da7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -39,7 +39,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaProducerTemplate implements TbQueueProducer { +public class TbKafkaProducerTemplate implements TbQueueProducer { private final KafkaProducer producer; @@ -54,7 +54,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro private final Set topics; @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId, TbQueueAdmin admin) { + private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 8682ca385f..4b68abf929 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.kafka; import lombok.Getter; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -24,6 +25,7 @@ import java.util.HashMap; import java.util.Map; @Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") public class TbKafkaTopicConfigs { @Value("${queue.kafka.topic-properties.core}") private String coreProperties; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 26da11aa3b..90fe643505 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -36,6 +36,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -49,14 +50,20 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings) { + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,69 +71,74 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 87aba51843..e169e6502d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -40,6 +40,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -52,70 +53,81 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index d83b760c19..33f1d8c113 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -48,54 +49,63 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings) { + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 4196b226ee..644bc6b709 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -35,6 +35,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -44,33 +45,38 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); this.serviceInfoProvider = serviceInfoProvider; + + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbAwsSqsProducerTemplate> producerTemplate = - new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); TbAwsSqsConsumerTemplate> consumerTemplate = - new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,17 +87,17 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 29660f2461..c03f5e52d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -33,6 +34,7 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -60,8 +62,15 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(topic -> { + + templateBuilder.queueAdmin(new TbQueueAdmin() { + @Override + public void createTopicIfNotExists(String topic) {} + + @Override + public void destroy() {} }); + templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 9eb656c2ea..72c6f7b8b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -37,9 +37,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -87,16 +87,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); @@ -106,7 +106,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -116,7 +116,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -126,7 +126,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -136,7 +136,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -147,7 +147,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { String queueName = configuration.getName(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); @@ -159,7 +159,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -171,7 +171,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); @@ -183,7 +183,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -195,7 +195,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); @@ -207,7 +207,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTransportApiResponseProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); @@ -218,13 +218,13 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 8800c1c55a..ca10830981 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -37,9 +37,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -82,16 +82,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -101,7 +101,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -111,7 +111,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -121,7 +121,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -131,7 +131,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -141,7 +141,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); @@ -153,7 +153,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -165,7 +165,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); @@ -177,7 +177,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTransportApiResponseProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -188,13 +188,13 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b96d4ebe86..bcac399a64 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -35,9 +35,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -76,15 +76,15 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -94,7 +94,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -104,7 +104,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -115,7 +115,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -125,7 +125,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -136,7 +136,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { String queueName = configuration.getName(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); @@ -148,7 +148,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -161,13 +161,13 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index f5a31cf1c7..2f7aab4779 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -30,9 +30,9 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -71,21 +71,21 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); requestBuilder.admin(transportApiAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); @@ -106,7 +106,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -116,7 +116,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -126,7 +126,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index f0af639d4d..cd575818f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -121,6 +121,11 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } + @Override + public void destroy() { + + } + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index fbd678045b..9a7e5db342 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -14,15 +14,14 @@ * limitations under the License. */ package org.thingsboard.server.queue.rabbitmq; + import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; - import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.concurrent.TimeoutException; @@ -62,8 +61,8 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { } } - @PreDestroy - private void destroy() { + @Override + public void destroy() { if (channel != null) { try { channel.close(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index 6080baec26..8e293e6dff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -21,45 +21,56 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.CreateQueueRequest; -import com.amazonaws.services.sqs.model.QueueAttributeName; import org.thingsboard.server.queue.TbQueueAdmin; -import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class TbAwsSqsAdmin implements TbQueueAdmin { - private final TbAwsSqsSettings sqsSettings; - private final Map attributes = new HashMap<>(); - private final AWSStaticCredentialsProvider credProvider; + private final Map attributes; + private final AmazonSQS sqsClient; + private final Set queues; - public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings) { - this.sqsSettings = sqsSettings; + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { + this.attributes = attributes; AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - this.credProvider = new AWSStaticCredentialsProvider(awsCredentials); + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .withRegion(sqsSettings.getRegion()) + .build(); - attributes.put("FifoQueue", "true"); - attributes.put("ContentBasedDeduplication", "true"); - attributes.put(QueueAttributeName.VisibilityTimeout.toString(), sqsSettings.getVisibilityTimeout()); + queues = sqsClient + .listQueues() + .getQueueUrls() + .stream() + .map(this::getQueueNameFromUrl) + .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); } @Override public void createTopicIfNotExists(String topic) { - AmazonSQS sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); + String queueName = topic.replaceAll("\\.", "_") + ".fifo"; + if (queues.contains(queueName)) { + return; + } + final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); + String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); + queues.add(getQueueNameFromUrl(queueUrl)); + } + + private String getQueueNameFromUrl(String queueUrl) { + int delimiterIndex = queueUrl.lastIndexOf("/"); + return queueUrl.substring(delimiterIndex + 1); + } - final CreateQueueRequest createQueueRequest = - new CreateQueueRequest(topic.replaceAll("\\.", "_") + ".fifo") - .withAttributes(attributes); - try { - sqsClient.createQueue(createQueueRequest); - } finally { - if (sqsClient != null) { - sqsClient.shutdown(); - } + @Override + public void destroy() { + if (sqsClient != null) { + sqsClient.shutdown(); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java new file mode 100644 index 0000000000..70a9587bef --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.services.sqs.model.QueueAttributeName; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +public class TbAwsSqsQueueAttributes { + @Value("${queue.aws-sqs.queue-properties.core}") + private String coreProperties; + @Value("${queue.aws-sqs.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.aws-sqs.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.aws-sqs.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.aws-sqs.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreAttributes; + @Getter + private Map ruleEngineAttributes; + @Getter + private Map transportApiAttributes; + @Getter + private Map notificationsAttributes; + @Getter + private Map jsExecutorAttributes; + + private final Map defaultAttributes = new HashMap<>(); + + @PostConstruct + private void init() { + defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); + defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); + + coreAttributes = getConfigs(coreProperties); + ruleEngineAttributes = getConfigs(ruleEngineProperties); + transportApiAttributes = getConfigs(transportApiProperties); + notificationsAttributes = getConfigs(notificationsProperties); + jsExecutorAttributes = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + validateAttributeName(key); + configs.put(key, value); + } + configs.putAll(defaultAttributes); + return configs; + } + + private void validateAttributeName(String key) { + QueueAttributeName.fromValue(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java index 42a2f1e983..922a2b1062 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -39,6 +39,4 @@ public class TbAwsSqsSettings { @Value("${queue.aws_sqs.threads_per_topic}") private int threadsPerTopic; - @Value("${queue.aws_sqs.visibility_timeout}") - private String visibilityTimeout; } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index e2fb7e149a..78dc191e2c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -62,7 +62,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index dfd2c14be2..82b548a456 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -63,7 +63,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 81f0fb795d..e3faf9cb87 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -93,7 +93,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" From 12973cecf1b05ca71d24d5545d60b74b6af3b176 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 12:51:05 +0300 Subject: [PATCH 057/102] added method destroy to admins and aws sqs factories --- .../provider/AwsSqsMonolithQueueFactory.java | 21 +++++++++++++++++++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++++++++++++++++++ .../AwsSqsTbRuleEngineQueueFactory.java | 18 ++++++++++++++++ .../provider/AwsSqsTransportQueueFactory.java | 12 +++++++++++ 4 files changed, 72 insertions(+) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 90fe643505..76ff04c238 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -39,6 +39,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -145,4 +147,23 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index e169e6502d..a7349091bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -43,6 +43,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -134,4 +136,23 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 33f1d8c113..3056eced2c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -114,4 +116,20 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 644bc6b709..c9ba3ec15f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -38,6 +38,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -100,4 +102,14 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } From adae8aeb30631d292573833d58d621ef08606a5a Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Fri, 17 Apr 2020 16:55:17 +0300 Subject: [PATCH 058/102] Improved implementaton of ProcessingAttemptContext to avoid synchronizations --- .../queue/ProcessingAttemptContext.java | 21 +++----- .../queue/ProcessingAttemptContextTest.java | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java index 2073ff8c21..aefb1697cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -27,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ProcessingAttemptContext { private final TbRuleEngineSubmitStrategy submitStrategy; + private final AtomicInteger pendingCount; private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); @Getter private final ConcurrentMap> pendingMap; @@ -45,6 +47,7 @@ public class ProcessingAttemptContext { public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); } public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { @@ -54,16 +57,12 @@ public class ProcessingAttemptContext { public void onSuccess(UUID id) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; successMap.put(id, msg); + submitStrategy.onSuccess(id); } - submitStrategy.onSuccess(id); if (empty) { processingTimeoutLatch.countDown(); } @@ -72,13 +71,9 @@ public class ProcessingAttemptContext { public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; failedMap.put(id, msg); exceptionsMap.putIfAbsent(tenantId, e); } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java new file mode 100644 index 0000000000..6c3914abc8 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -0,0 +1,51 @@ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ProcessingAttemptContextTest { + + @Test + public void testHighConcurrencyCase() throws InterruptedException { + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + int msgCount = 1000; + int parallelCount = 5; + ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); + try { + ConcurrentMap> messages = new ConcurrentHashMap<>(); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + when(strategyMock.getPendingMap()).thenReturn(messages); + ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + for (UUID uuid : messages.keySet()) { + for (int i = 0; i < parallelCount; i++) { + executorService.submit(() -> context.onSuccess(uuid)); + } + } + Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); + Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); + } finally { + executorService.shutdownNow(); + } + } +} From eae9d3c8e29ac276a1005c960951ac40535e6ad2 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Mon, 20 Apr 2020 10:55:25 +0300 Subject: [PATCH 059/102] Processing Atempt Context Improvement --- .../queue/ProcessingAttemptContextTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java index 6c3914abc8..2de2414f71 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; From b1f87206f3d930fbec7270ea018ba68a188315fc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 11:02:40 +0300 Subject: [PATCH 060/102] Minor improvements --- .../server/actors/ruleChain/DefaultTbContext.java | 3 +++ .../server/actors/ruleChain/RuleNodeActor.java | 2 ++ .../server/service/state/DefaultDeviceStateService.java | 8 ++++---- .../server/service/state/DeviceStateService.java | 2 +- .../common/transport/service/DefaultTransportService.java | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b268236448..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -192,6 +192,9 @@ class DefaultTbContext implements TbContext { @Override public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } tbMsg.getCallback().onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 64431fdf33..abb98468bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -54,6 +55,7 @@ public class RuleNodeActor extends ComponentActor> partitionedDevices = new ConcurrentHashMap<>(); - private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 16c699b38a..1665a27b84 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener callback.onSuccess(null)); } } From 341f0d14ab0ebe221b1ff2ea94bfa425980c03ab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 12:05:27 +0300 Subject: [PATCH 061/102] Fixed tests inconsistency due to new queues --- .../mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java | 4 ++++ .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 5ba4dfcbae..485d618a01 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -109,6 +109,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -165,6 +167,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 122bef2e26..7e329d8b05 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,7 +79,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(1000); + Thread.sleep(2000); List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); From 58d9c313a89d3929ad7c3d0e46ed287b91132355 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 16:12:03 +0300 Subject: [PATCH 062/102] Fixed tests --- .../DefaultRuleEngineStatisticsService.java | 11 +- .../controller/BaseAssetControllerTest.java | 128 ++++++++++-------- .../BaseEntityViewControllerTest.java | 6 +- .../controller/ControllerNoSqlTestSuite.java | 7 + .../controller/ControllerSqlTestSuite.java | 7 + .../server/mqtt/MqttNoSqlTestSuite.java | 7 + .../server/mqtt/MqttSqlTestSuite.java | 7 + .../rules/RuleEngineNoSqlTestSuite.java | 7 + .../server/rules/RuleEngineSqlTestSuite.java | 7 + .../server/system/SystemNoSqlTestSuite.java | 7 + .../server/system/SystemSqlTestSuite.java | 7 + .../server/queue/memory/InMemoryStorage.java | 42 +++--- .../queue/memory/InMemoryTbQueueConsumer.java | 11 +- tools/pom.xml | 2 +- 14 files changed, 173 insertions(+), 83 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 3951703351..a506b87ab0 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -47,6 +47,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { + public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final FutureCallback CALLBACK = new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { @@ -95,7 +96,13 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + try { + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + } catch (DataValidationException e2) { + if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e2; + } + } }); ruleEngineStats.reset(); } @@ -113,7 +120,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS asset = new Asset(); asset.setTenantId(tenantId); asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType("TbServiceQueue"); + asset.setType(TB_SERVICE_QUEUE); asset = assetService.saveAsset(asset); } assetId = asset.getId(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 56c73b5e13..0422a85416 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/"+savedAsset.getId().getId().toString()) + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/"+savedAsset.getId().getId().toString()) + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); + Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index e60f549610..420446e04d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -426,7 +426,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); -// client.disconnect(); + client.disconnect(); } private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { @@ -463,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - + client.disconnect(); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java index 4ad41824a0..781c483fc5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.controller; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false), new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 8dc0acff57..347eaee7eb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.controller; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index 9c4030cff4..7360c5c506 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 2863589ba1..70de3b27ca 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.mqtt; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -32,4 +34,9 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java index e44c383452..fbab13147c 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java @@ -16,11 +16,13 @@ package org.thingsboard.server.rules; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 5f930821f7..83ac5f7703 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.rules; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java index 41be7641c6..c4182db3ee 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.system; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index b12d513ce0..6060a2cc2b 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.system; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,4 +37,9 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index f919df871e..e119d1433b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -30,11 +30,9 @@ import java.util.concurrent.TimeUnit; public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; - private volatile boolean stopped; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); - stopped = false; } public static InMemoryStorage getInstance() { @@ -52,33 +50,31 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) { + public List get(String topic, long durationInMillis) throws InterruptedException { if (storage.containsKey(topic)) { - try { - List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); - if (first != null) { - entities = new ArrayList<>(); - entities.add(first); - List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 999); - for (TbQueueMsg other : otherList) { - entities.add((T) other); - } - } else { - entities = Collections.emptyList(); - } - return entities; - } catch (InterruptedException e) { - if (!stopped) { - log.warn("Queue was interrupted", e); + List entities; + T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 999); + for (TbQueueMsg other : otherList) { + entities.add((T) other); } + } else { + entities = Collections.emptyList(); } + return entities; } return Collections.emptyList(); } - public void stop() { - stopped = true; + /** + * Used primarily for testing. + */ + public void cleanup() { + storage.clear(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 2b81839540..3920f143f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -66,7 +66,16 @@ public class InMemoryTbQueueConsumer implements TbQueueCon if (subscribed) { List messages = partitions .stream() - .map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) + .map(tpi -> { + try { + return storage.get(tpi.getFullTopicName(), durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Queue was interrupted.", e); + } + return Collections.emptyList(); + } + }) .flatMap(List::stream) .map(msg -> (T) msg).collect(Collectors.toList()); if (messages.size() > 0) { diff --git a/tools/pom.xml b/tools/pom.xml index 64027c478e..1960d330d0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.4 + 3.11.6 com.datastax.cassandra From 1b9df18c4592ea16d35b0bf0eace8e1f20a2ca43 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 20:30:56 +0300 Subject: [PATCH 063/102] created TbPubSubSubscriptionSettings --- .../src/main/resources/thingsboard.yml | 7 +- .../provider/PubSubMonolithQueueFactory.java | 40 +++++++---- .../provider/PubSubTbCoreQueueFactory.java | 36 ++++++---- .../PubSubTbRuleEngineQueueFactory.java | 32 ++++++--- .../provider/PubSubTransportQueueFactory.java | 36 ++++++---- .../server/queue/pubsub/TbPubSubAdmin.java | 35 +++++++-- .../server/queue/pubsub/TbPubSubSettings.java | 3 - .../pubsub/TbPubSubSubscriptionSettings.java | 71 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 7 +- .../src/main/resources/tb-http-transport.yml | 7 +- .../src/main/resources/tb-mqtt-transport.yml | 7 +- 11 files changed, 219 insertions(+), 62 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e1296275cd..0cae3252ff 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -543,9 +543,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index b1afc61dbd..ba806c5daf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -38,6 +38,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -53,88 +54,99 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index edfcdc3d8e..eaffce241f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -47,71 +49,79 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 101f335536..3206f03250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -32,9 +32,11 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -46,59 +48,67 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index f15faa11da..2a5de01f86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -25,12 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -38,6 +34,11 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -50,33 +51,42 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producer); templateBuilder.responseTemplate(consumer); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -87,17 +97,17 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index cd575818f2..f3b6d3f10c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -19,32 +19,37 @@ import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.Duration; import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PushConfig; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @Slf4j public class TbPubSubAdmin implements TbQueueAdmin { + private static final String ACK_DEADLINE = "ackDeadlineInSec"; + private static final String MESSAGE_RETENTION = "messageRetentionInSec"; private final TbPubSubSettings pubSubSettings; private final SubscriptionAdminSettings subscriptionAdminSettings; private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + private final Map subscriptionProperties; - public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { + public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { this.pubSubSettings = pubSubSettings; + this.subscriptionProperties = subscriptionSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); @@ -149,8 +154,15 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - subscriptionAdminClient.createSubscription( - subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); + + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); + + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); subscriptionSet.add(subscriptionName.toString()); log.info("Created new subscription: [{}]", subscriptionName.toString()); } catch (IOException e) { @@ -159,4 +171,19 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } + private void setAckDeadline(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(ACK_DEADLINE)) { + builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); + } + } + + private void setMessageRetention(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { + Duration duration = Duration + .newBuilder() + .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) + .build(); + builder.setMessageRetentionDuration(duration); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index 851a2b3245..d6c6d0e271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -40,9 +40,6 @@ public class TbPubSubSettings { @Value("${queue.pubsub.service_account}") private String serviceAccount; - @Value("${queue.pubsub.ack_deadline}") - private int ackDeadline; - @Value("${queue.pubsub.max_msg_size}") private int maxMsgSize; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java new file mode 100644 index 0000000000..e98f51ec7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +public class TbPubSubSubscriptionSettings { + @Value("${queue.pubsub.queue-properties.core}") + private String coreProperties; + @Value("${queue.pubsub.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.pubsub.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.pubsub.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.pubsub.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreSettings; + @Getter + private Map ruleEngineSettings; + @Getter + private Map transportApiSettings; + @Getter + private Map notificationsSettings; + @Getter + private Map jsExecutorSettings; + + @PostConstruct + private void init() { + coreSettings = getSettings(coreProperties); + ruleEngineSettings = getSettings(ruleEngineProperties); + transportApiSettings = getSettings(transportApiProperties); + notificationsSettings = getSettings(notificationsProperties); + jsExecutorSettings = getSettings(jsExecutorProperties); + } + + private Map getSettings(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 78dc191e2c..c3e646c77c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -72,9 +72,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 82b548a456..0d48bda2b9 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -73,9 +73,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e3faf9cb87..4249757db1 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -103,9 +103,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" From 2ad4ddf1fbe0c1350e31a70e369fe96c0085bb84 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 21:05:12 +0300 Subject: [PATCH 064/102] pubsub improvements --- .../provider/PubSubMonolithQueueFactory.java | 21 ++++ .../provider/PubSubTbCoreQueueFactory.java | 18 ++++ .../PubSubTbRuleEngineQueueFactory.java | 18 ++++ .../provider/PubSubTransportQueueFactory.java | 18 ++++ .../server/queue/pubsub/TbPubSubAdmin.java | 101 +++++++++--------- 5 files changed, 124 insertions(+), 52 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index ba806c5daf..af0b276c1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -45,6 +45,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -153,4 +155,23 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index eaffce241f..0a23d58e46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -42,6 +42,8 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -128,4 +130,20 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 3206f03250..79501130ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -41,6 +41,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -117,4 +119,20 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 2a5de01f86..7e859e02ab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -111,4 +113,20 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index f3b6d3f10c..1241370d05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -40,9 +40,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { private static final String ACK_DEADLINE = "ackDeadlineInSec"; private static final String MESSAGE_RETENTION = "messageRetentionInSec"; + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + private final TbPubSubSettings pubSubSettings; - private final SubscriptionAdminSettings subscriptionAdminSettings; - private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); private final Map subscriptionProperties; @@ -51,6 +52,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { this.pubSubSettings = pubSubSettings; this.subscriptionProperties = subscriptionSettings; + TopicAdminSettings topicAdminSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -58,6 +60,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create TopicAdminSettings."); } + SubscriptionAdminSettings subscriptionAdminSettings; try { subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -65,7 +68,9 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create SubscriptionAdminSettings."); } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + try { + topicAdminClient = TopicAdminClient.create(topicAdminSettings); + ListTopicsRequest listTopicsRequest = ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); @@ -77,7 +82,8 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to get topics.", e); } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + try { + subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); ListSubscriptionsRequest listSubscriptionsRequest = ListSubscriptionsRequest.newBuilder() @@ -104,31 +110,21 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; } - - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); - createSubscriptionIfNotExists(partition, topicName); - } catch (IOException e) { - log.error("Failed to create topic: [{}].", topicName.toString(), e); - throw new RuntimeException("Failed to create topic.", e); } - } - - @Override - public void destroy() { + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); } private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { @@ -139,36 +135,27 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; } + } - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } catch (IOException e) { - log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); - throw new RuntimeException("Failed to create subscription.", e); - } + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); } private void setAckDeadline(Subscription.Builder builder) { @@ -186,4 +173,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { builder.setMessageRetentionDuration(duration); } } + + @Override + public void destroy() { + if (topicAdminClient != null) { + topicAdminClient.close(); + } + if (subscriptionAdminClient != null) { + subscriptionAdminClient.close(); + } + } } From c3bd2a0ebe2e7c005cae7520cc1ab6bf4e7224f8 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Sat, 18 Apr 2020 00:33:24 +0300 Subject: [PATCH 065/102] Fixed incorrect params in the get timeseries request --- .../thingsboard/rest/client/RestClient.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7e0dbf3858..f6a6541de0 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1646,7 +1646,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1998,17 +1998,25 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink) { - String urlParams = "limit={limit}&ascOrder={ascOrder}"; + return getUrlParams(pageLink, "startTime", "endTime"); + } + + private String getUrlParamsTs(TimePageLink pageLink) { + return getUrlParams(pageLink, "startTs", "endTs"); + } + + private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { + StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); if (pageLink.getStartTime() != null) { - urlParams += "&startTime={startTime}"; + urlParams.append("&").append(startTime).append("={startTime}"); } if (pageLink.getEndTime() != null) { - urlParams += "&endTime={endTime}"; + urlParams.append("&").append(endTime).append("={endTime}"); } if (pageLink.getIdOffset() != null) { - urlParams += "&offset={offset}"; + urlParams.append("&offset={offset}"); } - return urlParams; + return urlParams.toString(); } private String getUrlParams(TextPageLink pageLink) { From eab6ba459824a9916ab1438418fcc6447c119bcd Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Mon, 20 Apr 2020 19:38:59 +0300 Subject: [PATCH 066/102] StringBuilder replaced with String --- .../java/org/thingsboard/rest/client/RestClient.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index f6a6541de0..652de133b7 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2006,17 +2006,17 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { - StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); + String urlParams = "limit={limit}&ascOrder={ascOrder}"; if (pageLink.getStartTime() != null) { - urlParams.append("&").append(startTime).append("={startTime}"); + urlParams += "&" + startTime + "={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams.append("&").append(endTime).append("={endTime}"); + urlParams += "&" + endTime + "={endTime}"; } if (pageLink.getIdOffset() != null) { - urlParams.append("&offset={offset}"); + urlParams += "&offset={offset}"; } - return urlParams.toString(); + return urlParams; } private String getUrlParams(TextPageLink pageLink) { From 283ad27cb54d476e011b92e4de94aac465a89c39 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Apr 2020 17:08:51 +0300 Subject: [PATCH 067/102] added rabbitmq queue arguments --- .../src/main/resources/thingsboard.yml | 6 ++ .../provider/AwsSqsTransportQueueFactory.java | 11 ++- .../InMemoryTbTransportQueueFactory.java | 9 +- .../RabbitMqMonolithQueueFactory.java | 61 +++++++++--- .../provider/RabbitMqTbCoreQueueFactory.java | 57 ++++++++--- .../RabbitMqTbRuleEngineQueueFactory.java | 48 +++++++-- .../RabbitMqTransportQueueFactory.java | 50 ++++++++-- .../ServiceBusTransportQueueFactory.java | 6 +- .../queue/rabbitmq/TbRabbitMqAdmin.java | 11 +-- .../rabbitmq/TbRabbitMqQueueArguments.java | 98 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 6 ++ .../src/main/resources/tb-http-transport.yml | 6 ++ .../src/main/resources/tb-mqtt-transport.yml | 6 ++ 13 files changed, 319 insertions(+), 56 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cae3252ff..0b2c40acef 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index c9ba3ec15f..351cd4058c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -30,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -47,8 +48,10 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; @@ -56,12 +59,15 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @@ -94,7 +100,7 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override @@ -105,6 +111,9 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @PreDestroy private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } if (transportApiAdmin != null) { transportApiAdmin.destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index c03f5e52d8..2855e577ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -32,9 +32,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -43,13 +43,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -86,7 +89,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index ff4a69e2e6..72f45e5276 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -28,8 +28,10 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -37,6 +39,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -48,7 +52,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -56,7 +65,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,73 +73,97 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 5708e0738c..d84bbedbd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -34,13 +34,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -51,7 +55,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, TbQueueCoreSettings coreSettings, @@ -59,67 +68,91 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index e2755938d9..a18f427fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -48,55 +52,63 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @@ -105,4 +117,20 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index dc6154255c..841e004b3f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -30,12 +30,17 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -43,34 +48,45 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbQueueCoreSettings coreSettings, + TbRabbitMqQueueArguments queueArguments) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbRabbitMqProducerTemplate> producerTemplate = - new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); TbRabbitMqConsumerTemplate> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,17 +97,33 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index d38a4389a3..33c5277ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -29,6 +29,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -41,17 +42,20 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory private final TbServiceBusSettings serviceBusSettings; private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbQueueAdmin admin) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -82,7 +86,7 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 9a7e5db342..3bef6deb84 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,24 +18,23 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.concurrent.TimeoutException; @Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") public class TbRabbitMqAdmin implements TbQueueAdmin { private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; + private final Map arguments; - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { this.rabbitMqSettings = rabbitMqSettings; + this.arguments = arguments; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); @@ -55,7 +54,7 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String topic) { try { - channel.queueDeclare(topic, false, false, false, null); + channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { log.error("Failed to bind queue: [{}]", topic, e); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java new file mode 100644 index 0000000000..eb73e7dce6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqQueueArguments { + @Value("${queue.rabbitmq.queue-properties.core}") + private String coreProperties; + @Value("${queue.rabbitmq.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.rabbitmq.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.rabbitmq.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.rabbitmq.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreArgs; + @Getter + private Map ruleEngineArgs; + @Getter + private Map transportApiArgs; + @Getter + private Map notificationsArgs; + @Getter + private Map jsExecutorArgs; + + @PostConstruct + private void init() { + coreArgs = getArgs(coreProperties); + ruleEngineArgs = getArgs(ruleEngineProperties); + transportApiArgs = getArgs(transportApiProperties); + notificationsArgs = getArgs(notificationsProperties); + jsExecutorArgs = getArgs(jsExecutorProperties); + } + + private Map getArgs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } + return configs; + } + + private Object getObjectValue(String str) { + if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { + return Boolean.valueOf(str); + } else if (isNumeric(str)) { + return getNumericValue(str); + } + return str; + } + + private Object getNumericValue(String str) { + if (str.contains(".")) { + return Double.valueOf(str); + } else { + return Long.valueOf(str); + } + } + + private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); + + public boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + return PATTERN.matcher(strNum).matches(); + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3e646c77c..c3dcb76508 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -95,6 +95,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0d48bda2b9..baac6e977f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -96,6 +96,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 4249757db1..e920891d66 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -126,6 +126,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 53eb09bcb939ae28cbefefbbab5cb2dfe3d05434 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:31:25 +0300 Subject: [PATCH 068/102] Improvements to tests due to new architecture --- .../RuleChainActorMessageProcessor.java | 5 ++- .../server/actors/tenant/TenantActor.java | 40 ++++++++++++------- .../controller/TelemetryController.java | 8 ++-- .../queue/DefaultTbClusterService.java | 6 +++ .../DefaultTbRuleEngineConsumerService.java | 2 +- ...TbRuleEngineProcessingStrategyFactory.java | 16 ++++++-- .../rpc/DefaultTbCoreDeviceRpcService.java | 10 ++--- .../src/main/resources/thingsboard.yml | 2 +- ...tractMqttServerSideRpcIntegrationTest.java | 15 ++++--- .../AbstractMqttTelemetryIntegrationTest.java | 4 +- ...AbstractRuleEngineFlowIntegrationTest.java | 4 +- application/src/test/resources/logback.xml | 2 +- .../server/queue/memory/InMemoryStorage.java | 4 +- .../queue/memory/InMemoryTbQueueConsumer.java | 2 +- .../TbRuleEngineQueueConfiguration.java | 2 +- dao/src/test/resources/sql-test.properties | 10 ++++- 16 files changed, 87 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index a374e590b5..4e3d69c2b7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -249,7 +249,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - if (isRuleEngineForCurrentTenant) { - try { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; } - } catch (Exception e) { - cantFindTenant = true; } + log.info("[{}] Tenant actor started.", tenantId); } - log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -104,7 +111,12 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (cantFindTenant) { - log.info("Missing Tenant msg: {}", msg); + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; } switch (msg.getMsgType()) { case PARTITION_CHANGE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 8352e3f457..aee56652c4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -344,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -354,13 +354,13 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); @@ -397,7 +397,7 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( user.getTenantId(), deviceId, scope, attributes), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f0054771d9..926b08f38b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -87,6 +87,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); byte[] msgBytes = encodingService.encode(msg); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); @@ -96,6 +97,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -108,6 +110,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); toRuleEngineMsgs.incrementAndGet(); } @@ -123,6 +126,7 @@ public class DefaultTbClusterService implements TbClusterService { } } TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -134,6 +138,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -147,6 +152,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); toTransportNfs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index c8f126b26c..6736518bcc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -174,7 +174,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< })); boolean timeout = false; - if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { timeout = true; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index b6579b8dcb..bbf283e962 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; @@ -32,7 +34,7 @@ public class TbRuleEngineProcessingStrategyFactory { public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { switch (configuration.getType()) { - case "SKIP_ALL": + case "SKIP_ALL_FAILURES": return new SkipStrategy(name); case "RETRY_ALL": return new RetryStrategy(name, true, true, true, configuration); @@ -98,7 +100,7 @@ public class TbRuleEngineProcessingStrategyFactory { } log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } if (pauseBetweenRetries > 0) { try { @@ -123,7 +125,15 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + if (!result.isSuccess()) { + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index 8a1423f97c..f6e5eb8b43 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -106,7 +106,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { @Override public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -177,9 +177,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to rule engine request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for processing to rule engine request.", requestId); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); @@ -189,9 +189,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to device request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for to device request.", requestId); localToDeviceRpcRequests.remove(requestId); }, timeout, TimeUnit.MILLISECONDS); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0b2c40acef..3df3cb9559 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -622,7 +622,7 @@ queue: # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 485d618a01..9ddb2c81d9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; + private static final AtomicInteger atomicInteger = new AtomicInteger(2); + @Before public void beforeTest() throws Exception { @@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -130,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -139,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -169,7 +172,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC Thread.sleep(2000); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -187,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -196,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 7e329d8b05..f96f207834 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -107,13 +107,13 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); - client.connect(options).waitForCompletion(3000); + client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport - Thread.sleep(1000); + Thread.sleep(5000); doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 3343d271fe..87a181deae 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -152,7 +152,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -265,7 +265,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..7943b9d9b1 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index e119d1433b..f51f5b8ae0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -50,10 +50,10 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) throws InterruptedException { + public List get(String topic) throws InterruptedException { if (storage.containsKey(topic)) { List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + T first = (T) storage.get(topic).poll(); if (first != null) { entities = new ArrayList<>(); entities.add(first); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 3920f143f1..a619eda132 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -68,7 +68,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon .stream() .map(tpi -> { try { - return storage.get(tpi.getFullTopicName(), durationInMillis); + return storage.get(tpi.getFullTopicName()); } catch (InterruptedException e) { if (!stopped) { log.error("Queue was interrupted.", e); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index c5a24f20c6..019a76953b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -24,7 +24,7 @@ public class TbRuleEngineQueueConfiguration { private String topic; private int pollInterval; private int partitions; - private String packProcessingTimeout; + private long packProcessingTimeout; private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 13c0fbc818..7b5f82da2a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -35,4 +35,12 @@ service.type=monolith #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 + +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file From b2bb39f7a7c65c7e1ce8ee90ad17dcc6857ccf6e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:47:06 +0300 Subject: [PATCH 069/102] Restored WARN log level for tests --- application/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 7943b9d9b1..47dacce343 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + From 076cf655f87a914d1b882d35647faedaa8bd7c2e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 23:54:26 +0300 Subject: [PATCH 070/102] Improved tests --- .../thingsboard/server/controller/TelemetryController.java | 5 ----- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- dao/src/test/resources/sql-test.properties | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index aee56652c4..bb866e3e9b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -397,11 +397,6 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { - DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes), null); - } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f96f207834..0fe85fbb88 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -109,7 +109,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr client.setCallback(callback); client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"value\"}"; + String payload = "{\"key\":\"uniqueValue\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 7b5f82da2a..50d3a3feb3 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -37,6 +37,9 @@ service.type=monolith #spring.datasource.driverClassName=org.postgresql.Driver #spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 + queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main queue.rule-engine.queues[0].poll-interval=25 From dc3d1dd794f1c70cbd6a6eecc9a14aaf7abe45e4 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 13:03:53 +0300 Subject: [PATCH 071/102] added ttl for events --- .../main/data/upgrade/2.4.3/schema_update_ttl.sql | 15 +++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 5 +++++ .../service/ttl/events/EventsCleanUpService.java | 5 ++--- .../server/dao/model/ModelConstants.java | 3 +++ .../server/dao/model/sql/EventEntity.java | 12 ++++++++++++ dao/src/main/resources/sql/schema-entities.sql | 15 +++++++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index ff1fb5129b..bd40434ffe 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,3 +123,18 @@ BEGIN END LOOP; END $$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ + DECLARE + ttl_ts bigint; +BEGIN + IF system_ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; + diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..d7454db0b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index bb7d112bb1..e10ea30893 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - log.info("ttl: [{}]", ttl); - log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); - // TODO: 4/15/20 Do a clean up. + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..4a2f8aef23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,6 +32,9 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units + public static final long EPOCH_DIFF = 122192928000000000L; + /** * Generic constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..c0ad279d5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -37,6 +37,9 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; @@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data @EqualsAndHashCode(callSuper = true) @@ -73,9 +77,13 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; From 0103da1ba4852f305606c7eb52c4ffd560f78dbc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:12:57 +0300 Subject: [PATCH 072/102] Improved Callback and Destroy of ndoes --- .../queue/DefaultTbCoreConsumerService.java | 15 +++- .../DefaultTbRuleEngineConsumerService.java | 8 +- .../service/queue/TbMsgPackCallback.java | 8 +- ...t.java => TbMsgPackProcessingContext.java} | 4 +- .../server/service/queue/TbPackCallback.java | 28 ++---- .../queue/TbPackProcessingContext.java | 90 +++++++++++++++++++ .../processing/AbstractConsumerService.java | 11 ++- .../TbRuleEngineProcessingResult.java | 6 +- ...va => TbMsgPackProcessingContextTest.java} | 4 +- 9 files changed, 124 insertions(+), 50 deletions(-) rename application/src/main/java/org/thingsboard/server/service/queue/{ProcessingAttemptContext.java => TbMsgPackProcessingContext.java} (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java rename application/src/test/java/org/thingsboard/server/service/queue/{ProcessingAttemptContextTest.java => TbMsgPackProcessingContextTest.java} (94%) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c943e4c4d0..a5abd77ad5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -98,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToSubscriptionMgrMsg()) { @@ -147,8 +154,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } mainConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 6736518bcc..0c48c54412 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -64,9 +63,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -116,10 +112,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @PreDestroy public void stop() { + super.destroy(); if (submitExecutor != null) { submitExecutor.shutdownNow(); } - ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); } @@ -156,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< submitStrategy.init(msgs); while (!stopped) { - ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java index 2a6b6a658d..f093cc885a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -18,21 +18,17 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeException; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbMsgPackCallback implements TbMsgCallback { private final UUID id; private final TenantId tenantId; - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { + public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { this.id = id; this.tenantId = tenantId; this.ctx = ctx; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java index aefb1697cd..7e78ac6f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java @@ -29,7 +29,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -public class ProcessingAttemptContext { +public class TbMsgPackProcessingContext { private final TbRuleEngineSubmitStrategy submitStrategy; @@ -44,7 +44,7 @@ public class ProcessingAttemptContext { @Getter private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); this.pendingCount = new AtomicInteger(pendingMap.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index ba5a883ea0..5a5c172ee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -19,44 +19,26 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbPackCallback implements TbCallback { - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap failedMap; + private final TbPackProcessingContext ctx; private final UUID id; - public TbPackCallback(UUID id, - CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap failedMap) { + public TbPackCallback(UUID id, TbPackProcessingContext ctx) { this.id = id; - this.processingTimeoutLatch = processingTimeoutLatch; - this.ackMap = ackMap; - this.failedMap = failedMap; + this.ctx = ctx; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - T msg = ackMap.remove(id); - if (msg != null && ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onSuccess(id); } @Override public void onFailure(Throwable t) { log.trace("[{}] ON FAILURE", id, t); - T msg = ackMap.remove(id); - if (msg != null) { - failedMap.put(id, msg); - } - if (ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onFailure(id, t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java new file mode 100644 index 0000000000..e9f1224625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbPackProcessingContext { + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + + public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.processingTimeoutLatch = processingTimeoutLatch; + this.pendingCount = new AtomicInteger(ackMap.size()); + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + } + if (empty) { + processingTimeoutLatch.countDown(); + } else { + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T t : ackMap.values()) { + log.trace("left item: {}", t); + } + } + } + } + + public void onFailure(UUID id, Throwable t) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T v : ackMap.values()) { + log.trace("left item: {}", v); + } + } + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public ConcurrentMap getAckMap() { + return ackMap; + } + + public ConcurrentMap getFailedMap() { + return failedMap; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index a238e34337..c2705fcbdc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,11 +23,13 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.TbPackCallback; +import org.thingsboard.server.service.queue.TbPackProcessingContext; import javax.annotation.PreDestroy; import java.util.List; @@ -92,11 +94,12 @@ public abstract class AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { handleNotification(id, msg, callback); } catch (Throwable e) { @@ -105,8 +108,8 @@ public abstract class AbstractConsumerService log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); } nfConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index 8e0fcaa74a..e818ffb9d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.ProcessingAttemptContext; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import java.util.UUID; import java.util.concurrent.ConcurrentMap; @@ -32,9 +32,9 @@ public class TbRuleEngineProcessingResult { @Getter private final boolean timeout; @Getter - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { + public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { this.timeout = timeout; this.ctx = ctx; this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java similarity index 94% rename from application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java rename to application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java index 2de2414f71..43cd62ea97 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -37,7 +37,7 @@ import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) -public class ProcessingAttemptContextTest { +public class TbMsgPackProcessingContextTest { @Test public void testHighConcurrencyCase() throws InterruptedException { @@ -51,7 +51,7 @@ public class ProcessingAttemptContextTest { messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); } when(strategyMock.getPendingMap()).thenReturn(messages); - ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); for (UUID uuid : messages.keySet()) { for (int i = 0; i < parallelCount; i++) { executorService.submit(() -> context.onSuccess(uuid)); From 26a200dcfb1b610be6091b34edf7e76694ec8a3b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:27:16 +0300 Subject: [PATCH 073/102] Renaming the queues --- .../server/controller/QueueController.java | 2 +- .../TbRuleEngineSubmitStrategyFactory.java | 4 ++-- .../src/main/resources/thingsboard.yml | 19 +++++++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 8d1cec7f4b..a0b46f9166 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -43,7 +43,7 @@ public class QueueController extends BaseController { ServiceType type = ServiceType.valueOf(serviceType); switch (type) { case TB_RULE_ENGINE: - return Arrays.asList("HighPriority", "Main"); + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); default: return Collections.emptyList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java index f5a7457c17..90eb8561a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -29,9 +29,9 @@ public class TbRuleEngineSubmitStrategyFactory { return new BurstTbRuleEngineSubmitStrategy(name); case "BATCH": return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); - case "SEQUENTIAL_WITHIN_ORIGINATOR": + case "SEQUENTIAL_BY_ORIGINATOR": return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL_WITHIN_TENANT": + case "SEQUENTIAL_BY_TENANT": return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); case "SEQUENTIAL": return new SequentialTbRuleEngineSubmitStrategy(name); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3df3cb9559..9949a84c4d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -618,7 +618,7 @@ queue: partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: @@ -633,7 +633,7 @@ queue: partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch processing-strategy: @@ -642,6 +642,21 @@ queue: retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" From 71bf15caf79e7cd25f36227e969bf7910b3e0fc0 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:16:58 +0300 Subject: [PATCH 074/102] improvements --- .../data/upgrade/2.4.3/schema_update_ttl.sql | 24 ++++++++++++----- .../ttl/events/EventsCleanUpService.java | 7 +++-- .../src/main/resources/thingsboard.yml | 9 ++++--- .../resources/sql/schema-entities-hsql.sql | 27 +++++++++++++++++++ .../main/resources/sql/schema-entities.sql | 21 +++++++++++---- 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index bd40434ffe..dda3bd7b1c 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -124,17 +124,27 @@ BEGIN END $$; -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ - DECLARE - ttl_ts bigint; +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; - diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index e10ea30893..a608ca257b 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -31,9 +31,12 @@ import java.sql.SQLException; @Service public class EventsCleanUpService extends AbstractCleanUpService { - @Value("${sql.ttl.events.events_key_value_ttl}") + @Value("${sql.ttl.events.events_ttl}") private long ttl; + @Value("${sql.ttl.events.debug_events_ttl}") + private long debugTtl; + @Value("${sql.ttl.events.enabled}") private boolean ttlTaskExecutionEnabled; @@ -50,7 +53,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6eee7bc32a..4b584ab44f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,12 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: @@ -371,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index f28f7f5ebd..d4900020c6 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -251,3 +252,29 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a70eb13ab7..931856e1df 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint, + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -253,16 +253,27 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ DECLARE ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; From a0b4a46c3da516c1405932128ac87e3f770b7797 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:20:23 +0300 Subject: [PATCH 075/102] fix typo --- application/src/main/resources/thingsboard.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4b584ab44f..cb343e8bdb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,13 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: From 005d70d2eaf4051a28168c651d809574c9c98f6c Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:22:10 +0300 Subject: [PATCH 076/102] fix yml typo --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cb343e8bdb..456761a0cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -372,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: From ce086f915ea272504ce756c8e1c018a772c7a28b Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:53:38 +0300 Subject: [PATCH 077/102] try to pass travis --- .../thingsboard/server/install/ThingsboardInstallService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 842550d209..8dd45bc41b 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; From 6d20ed05501f4b2f42a5922f6f79a5f7b2af95d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 18:09:47 +0300 Subject: [PATCH 078/102] fix event insert repository --- .../event/AbstractEventInsertRepository.java | 3 ++- .../sql/event/HsqlEventInsertRepository.java | 4 +-- .../sql/event/PsqlEventInsertRepository.java | 2 +- .../resources/sql/schema-entities-hsql.sql | 25 ------------------- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- 6 files changed, 7 insertions(+), 31 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index b8d3af0968..8a341e0f81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -75,7 +75,8 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()); + .setParameter("tenant_id", entity.getTenantId()) + .setParameter("ts", entity.getTs()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index bebfd7c5e0..f5a00fe7f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -44,7 +44,7 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java index 1a8c12cb45..937262241a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java @@ -48,6 +48,6 @@ public class PsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) { - return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, " + updateKeyStatement + " returning *"; + return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, ts = :ts," + updateKeyStatement + " returning *"; } } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index d4900020c6..697e34930a 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -253,28 +253,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; - diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 32e2a78620..bb0a964c13 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3789444106..f83e80931b 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS From 5dbf7d764cceabb160fc168b579d4819c09e5b71 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 18:52:34 +0300 Subject: [PATCH 079/102] Default Dashboards --- .../data/json/demo/dashboards/gateways.json | 1274 +++++++++++++++++ .../dashboards/rule_engine_statistics.json | 520 +++++++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../DefaultTelemetryWebSocketService.java | 8 +- .../src/main/resources/thingsboard.yml | 15 +- .../TbSynchronizationBeginNode.java | 2 +- .../transaction/TbSynchronizationEndNode.java | 2 +- 7 files changed, 1811 insertions(+), 12 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/gateways.json create mode 100644 application/src/main/data/json/demo/dashboards/rule_engine_statistics.json diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json new file mode 100644 index 0000000000..f6370696c1 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -0,0 +1,1274 @@ +{ + "title": "Gateways", + "configuration": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "isSystemType": true, + "bundleAlias": "entity_admin_widgets", + "typeAlias": "device_admin_table2", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "entitiesTitle": "List of gateways", + "enableSelectColumnDisplay": true, + "displayEntityLabel": false, + "entityNameColumnTitle": "Gateway Name" + }, + "title": "Devices gateway table", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.3646047595211721 + }, + { + "name": "eventsSent", + "type": "timeseries", + "label": "Sent", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.7235710720767985 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Events", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5085933386303254 + }, + { + "name": "LOGS", + "type": "timeseries", + "label": "Latest log", + "color": "#ffc107", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3504240371585048, + "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" + }, + { + "name": "RemoteLoggingLevel", + "type": "attribute", + "label": "Log level", + "color": "#607d8b", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9785994222542516 + } + ], + "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" + } + ], + "showTitleIcon": true, + "titleIcon": "list", + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "List device", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", + "name": "Add device", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n\n", + "customCss": "", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "78845501-234e-a452-6819-82b5b776e99f", + "name": "Configuration", + "icon": "settings", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "openRightLayout": false, + "setEntityId": true + }, + { + "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", + "name": "Graphs", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname_grafic", + "setEntityId": true + }, + { + "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", + "name": "Edit device", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + }, + { + "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", + "name": "Delete device", + "icon": "delete", + "type": "custom", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" + } + ], + "rowClick": [ + { + "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", + "name": "to_entityname__config", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "setEntityId": true, + "stateEntityParamName": "" + } + ] + } + }, + "id": "94715984-ae74-76e4-20b7-2f956b01ed80" + }, + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 14, + "sizeY": 13, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "LOGS", + "type": "timeseries", + "label": "LOGS", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3496649158709739, + "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 2592000000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Debug events (logs)", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" + }, + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "opcuaEventsProduced", + "type": "timeseries", + "label": "opcuaEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.1477920581839779 + }, + { + "name": "opcuaEventsSent", + "type": "timeseries", + "label": "opcuaEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6500957113784758 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 7, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "Events", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.8156044798125357 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Produced", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.6538259344015449 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 604800000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 6, + "hideEmptyLines": true + }, + "title": "Total Messages", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": false, + "showMax": false, + "showAvg": true, + "showTotal": false + } + }, + "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "eventsSent", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.41414001784591314 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "eventsProduced", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.7819101846284422 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "History information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "bleEventsProduced", + "type": "timeseries", + "label": "bleEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.5625165504526104 + }, + { + "name": "bleEventsSent", + "type": "timeseries", + "label": "bleEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6817950080745288 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 5000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "attributes_card", + "type": "latest", + "title": "New widget", + "sizeX": 7, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "eventsTitle": "Gateway Events Form", + "eventsReg": [ + "EventsProduced", + "EventsSent" + ] + }, + "title": "Gateway events", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "466f046d-6005-a168-b107-60fcb2469cd5" + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "config_form_latest", + "type": "latest", + "title": "New widget", + "sizeX": 10, + "sizeY": 9, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "gatewayTitle": "Gateway configuration (Single device)", + "readOnly": false + }, + "title": "New Gateway configuration (Single device)", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodParams": "{}", + "methodName": "gateway_reboot", + "buttonText": "GATEWAY REBOOT" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodName": "gateway_restart", + "methodParams": "{}", + "buttonText": "gateway restart" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.9743324774725604 + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.5530093635101525 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [], + "useMarkerImageFunction": false, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.Mapnik", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5 + }, + "title": "Gateway Location", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" + } + }, + "states": { + "main_gateway": { + "name": "Gateways", + "root": true, + "layouts": { + "main": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname__config": { + "name": "${entityName} Configuration", + "root": false, + "layouts": { + "main": { + "widgets": { + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "sizeX": 14, + "sizeY": 13, + "row": 0, + "col": 10 + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "sizeX": 10, + "sizeY": 9, + "row": 0, + "col": 0 + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "sizeX": 4, + "sizeY": 2, + "row": 9, + "col": 0 + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "sizeX": 4, + "sizeY": 2, + "row": 11, + "col": 0 + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "sizeX": 6, + "sizeY": 4, + "row": 9, + "col": 4 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname_grafic": { + "name": "${entityName} Details", + "root": false, + "layouts": { + "main": { + "widgets": { + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 4, + "col": 7 + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "sizeX": 7, + "sizeY": 7, + "row": 5, + "col": 0 + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 0, + "col": 7 + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 8, + "col": 7 + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "sizeX": 7, + "sizeY": 5, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "auto 100%", + "autoFillHeight": true, + "mobileAutoFillHeight": true, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "3e0f533a-0db1-3292-184f-06e73535061a": { + "id": "3e0f533a-0db1-3292-184f-06e73535061a", + "alias": "Gateways", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "gateway", + "deviceNameFilter": "" + } + }, + "b2487e75-2fa4-f211-142c-434dfd50c70c": { + "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", + "alias": "Current Gateway", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "settings": { + "stateControllerId": "entity", + "showTitle": true, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true, + "titleColor": "rgba(0,0,0,0.870588)" + } + }, + "name": "Gateways" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json new file mode 100644 index 0000000000..b8231ab880 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -0,0 +1,520 @@ +{ + "title": "Rule Engine Statistics", + "configuration": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "successfulMsgs", + "type": "timeseries", + "label": "${entityName} Successful", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.15490750967648736 + }, + { + "name": "failedMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Failures", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.4186621166514697 + }, + { + "name": "tmpFailed", + "type": "timeseries", + "label": "${entityName} Processing Failures", + "color": "#ffc107", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.49891007198715376 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Queue Stats", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 24, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Chain", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleChainName;" + }, + "_hash": 0.9954481282345906 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Node", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleNodeName;" + }, + "_hash": 0.18580357036589978 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Latest Error", + "color": "#f44336", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).message;" + }, + "_hash": 0.7255162989552142 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Exceptions", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "timeoutMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Timeouts", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.565222981550328 + }, + { + "name": "tmpTimeout", + "type": "timeseries", + "label": "${entityName} Processing Timeouts", + "color": "#9c27b0", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.2679547062508352 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Processing Failures and Timeouts", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" + } + }, + "states": { + "default": { + "name": "Rule Engine Statistics", + "root": true, + "layouts": { + "main": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 0 + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "sizeX": 24, + "sizeY": 5, + "row": 7, + "col": 0 + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 12 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "alias": "TbServiceQueues", + "filter": { + "type": "assetType", + "resolveMultiple": true, + "assetType": "TbServiceQueue", + "assetNameFilter": "" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1586176634823, + "endTimeMs": 1586263034823 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Rule Engine Statistics" +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 78319997a9..c362c5e889 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -79,7 +79,7 @@ public class TbCoreConsumerStats { public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + " deviceState [{}] subMgr [{}] coreNfs [{}]", total, sessionEventCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 5e9e83b8aa..6afcaf97bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -345,6 +345,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -353,7 +355,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(false) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } @@ -440,6 +442,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -448,7 +452,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(true) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9949a84c4d..6a16349c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -410,8 +410,9 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: @@ -589,7 +590,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic @@ -615,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -630,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -643,9 +644,9 @@ queue: failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 31501688eb..751b061666 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -33,7 +33,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Starts synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index ac3d42ef36..8b83bff1fa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,7 +35,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Stops synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") From 087ea2d691b900d4f8cafc9dd055f7cad7019137 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Apr 2020 09:25:25 +0300 Subject: [PATCH 080/102] Commented Unstable Test --- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 0fe85fbb88..47e9537ef9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -97,7 +97,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - @Test + +// @Test - Unstable public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); From 9a052ef0bc333c8864b6ac9bc228d6290e0629d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Thu, 23 Apr 2020 12:39:33 +0300 Subject: [PATCH 081/102] fix typo in EventEntity class --- .../java/org/thingsboard/server/dao/model/sql/EventEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index 0a541e24b0..3790577144 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -113,6 +113,6 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity Date: Thu, 23 Apr 2020 14:10:45 +0300 Subject: [PATCH 082/102] Changed default partition sizes --- application/src/main/resources/thingsboard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6a16349c4c..7457cc8c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -616,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -631,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -646,7 +646,7 @@ queue: - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL From 1510fe8448002e5c0f73a7ed957a0d52721df8dc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:22:34 +0300 Subject: [PATCH 083/102] Default Dashboard and Rule Chain --- .../json/demo/dashboards/theromstats.json | 1236 +++++++++++++++++ .../demo/rule_chains/root_rule_chain.json | 146 ++ .../demo/rule_chains/thermostat_alarms.json | 137 ++ .../DefaultSystemDataLoaderService.java | 36 +- .../service/install/InstallScripts.java | 53 +- 5 files changed, 1604 insertions(+), 4 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/theromstats.json create mode 100644 application/src/main/data/json/demo/rule_chains/root_rule_chain.json create mode 100644 application/src/main/data/json/demo/rule_chains/thermostat_alarms.json diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/theromstats.json new file mode 100644 index 0000000000..bd191a6bc5 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/theromstats.json @@ -0,0 +1,1236 @@ +{ + "title": "Thermostats", + "configuration": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "entities_table", + "type": "latest", + "title": "New widget", + "sizeX": 11, + "sizeY": 11, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "enableSelectColumnDisplay": false, + "entitiesTitle": "Thermostats", + "displayEntityLabel": false, + "entityNameColumnTitle": "Thermostat name" + }, + "title": "Thermostats", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value === \"true\") {\n color = 'rgb(39, 134, 34)';\n} else {\n color = 'rgb(255, 0, 0)';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.9264526512320641 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9801965063904188, + "units": "°C", + "decimals": 1 + }, + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5726727868178358, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "85b803db-90f2-5c63-1388-a378e0eb10d6", + "name": "Edit location", + "icon": "map", + "type": "openDashboardState", + "targetDashboardStateId": "map", + "setEntityId": false + }, + { + "id": "8ab5a518-67d2-b6a2-956d-81fd512294b2", + "name": "Add", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add thermostat

\n \n \n \n \n
\n \n \n
\n \n \n \n
\n
Thermostat name is required.
\n
\n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id);\n updateAliasData();\n $mdDialog.hide();\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + } + ], + "actionCellButton": [ + { + "id": "ca241cd8-788d-5508-a9ce-74b03ef42a7f", + "name": "Chart", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + }, + { + "id": "7506576f-87ba-d3a0-88fb-e304d451776d", + "name": "Edit", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit thermostat {{vm.entityName}}

\n \n \n \n \n
\n \n \n
\n \n \n \n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Save\n Cancel\n \n
\n", + "customCss": ".edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n attributeService = $injector.get('attributeService');\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditEntityDialogController($scope,$mdDialog) {\n var vm = this;\n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.entityType = entityId.entityType;\n vm.attributes = {};\n vm.serverAttributes = {};\n getEntityInfo();\n \n vm.save = function() {\n saveAttributes();\n $mdDialog.hide();\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n vm.serverAttributes = angular.copy(vm.attributes);\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n function(data){\n if (data.length) {\n getEntityAttributes(data);\n }\n });\n }\n \n function saveAttributes() {\n var attributesArray = [];\n for (var key in vm.attributes) {\n if (vm.attributes[key] !== vm.serverAttributes[key]) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n}" + }, + { + "id": "3488848b-e47d-6af6-659f-5d78369ece5e", + "name": "Delete", + "icon": "delete", + "type": "custom", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n deviceService = $injector.get('deviceService')\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteEntityDialog();\n\nfunction openDeleteEntityDialog() {\n var title = 'Delete thermostat \"' + entityName + '\"';\n var content = 'Are you sure you want to delete the thermostat \"' +\n entityName + '\"?';\n var confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(function() {\n deleteEntity();\n })\n}\n\nfunction deleteEntity() {\n deviceService.deleteDevice(entityId.id).then(\n function success() {\n updateAliasData();\n },\n function fail() {\n showErrorDialog();\n }\n );\n}\n\nfunction updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}\n\nfunction showErrorDialog() {\n var title = 'Error';\n var content = 'An error occurred while deleting the thermostat. Please try again.';\n var alert = $mdDialog.alert()\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .parent(angular.element($document[0].body))\n .targetEvent($event)\n .multiple(true)\n .clickOutsideToClose(true)\n .ok('CLOSE');\n $mdDialog.show(alert);\n}" + } + ], + "rowClick": [ + { + "id": "e3928f23-c135-0766-71d5-65ed61e0ce8d", + "name": "show alarm", + "icon": "more_horiz", + "type": "updateDashboardState", + "targetDashboardStateId": "default", + "setEntityId": true, + "stateEntityParamName": "alarm" + } + ] + } + }, + "id": "f33c746c-0dfc-c212-395b-b448c8a17209" + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "isSystemType": true, + "bundleAlias": "alarm_widgets", + "typeAlias": "alarms_table", + "type": "alarm", + "title": "New widget", + "sizeX": 13, + "sizeY": 5, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSelection": true, + "enableSearch": true, + "displayDetails": true, + "allowAcknowledgment": true, + "allowClear": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "-createdTime", + "enableSelectColumnDisplay": false, + "enableStatusFilter": true, + "alarmsTitle": "Alarms" + }, + "title": "New Alarms table", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "alarmSource": { + "type": "entity", + "dataKeys": [ + { + "name": "createdTime", + "type": "alarm", + "label": "Created time", + "color": "#2196f3", + "settings": {}, + "_hash": 0.7308410188824108 + }, + { + "name": "originator", + "type": "alarm", + "label": "Originator", + "color": "#4caf50", + "settings": {}, + "_hash": 0.056085530105439485 + }, + { + "name": "type", + "type": "alarm", + "label": "Type", + "color": "#f44336", + "settings": {}, + "_hash": 0.10212012352561795 + }, + { + "name": "severity", + "type": "alarm", + "label": "Severity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1777349980531262 + }, + { + "name": "status", + "type": "alarm", + "label": "Status", + "color": "#607d8b", + "settings": {}, + "_hash": 0.7977920750136249 + } + ], + "entityAliasId": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "name": "alarms" + }, + "alarmSearchStatus": "ANY", + "alarmsPollingInterval": 5, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": {}, + "datasources": [], + "alarmsMaxCountLoad": 0, + "alarmsFetchSize": 100 + }, + "id": "7943196b-eedb-d422-f9c3-b32d379ad172" + }, + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.7852346160709658, + "units": "°C", + "decimals": 1 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Temperature", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "14a19183-f0b2-d6be-0f62-9863f0a51111" + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.28640715926957183, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Humidity", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "07f49fd5-a73b-d74c-c220-362c20af81f4" + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "update_multiple_attributes", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "alarmTemperature", + "type": "attribute", + "label": "High temperature alarm", + "color": "#4caf50", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.8725278440159361 + }, + { + "name": "thresholdTemperature", + "type": "attribute", + "label": "High temperature threshold, °C", + "color": "#f44336", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmTemperature" + }, + "_hash": 0.7316078472857874 + }, + { + "name": "alarmHumidity", + "type": "attribute", + "label": "Low humidity alarm", + "color": "#ffc107", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.5339673667431057 + }, + { + "name": "thresholdHumidity", + "type": "attribute", + "label": "Low humidity threshold, %", + "color": "#607d8b", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmHumidity" + }, + "_hash": 0.2687091190358901 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showActionButtons": false, + "showResultMessage": true, + "fieldsAlignment": "column", + "fieldsInRow": 2, + "groupTitle": "${entityName}", + "widgetTitle": "Termostat settings" + }, + "title": "New Update Multiple Attributes", + "dropShadow": true, + "enableFullscreen": false, + "enableDataExport": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "c4631f94-2db3-523b-4d09-2a1a0a75d93f" + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "isSystemType": true, + "bundleAlias": "maps_v2", + "typeAlias": "openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 13, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#2196f3", + "settings": {}, + "_hash": 0.1371919646686739, + "decimals": 1, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#4caf50", + "settings": {}, + "_hash": 0.043177186765847475, + "decimals": 0, + "postFuncBody": "return value || \"\";" + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#f44336", + "settings": {}, + "_hash": 0.5548964320315584 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1803778014971602 + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.30926987994082844 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Thermostat details
", + "markerImageSize": 48, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "colorFunction": "\n", + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "defaultCenterPosition": [ + 0, + 0 + ], + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "useLabelFunction": true, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "defaultZoomLevel": 14, + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;" + }, + "title": "Thermostat maps", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "headerButton": [], + "tooltipAction": [ + { + "id": "bef25673-b37a-8821-bc0f-5d6dd3680f24", + "name": "navigate_to_details", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb" + }, + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "00fb2742-ba1f-7e43-673f-d6c08b72ed06" + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "0a430429-9078-9ae6-2b67-e4a15a2bf8bf" + } + }, + "states": { + "default": { + "name": "Thermostat", + "root": true, + "layouts": { + "main": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "sizeX": 11, + "sizeY": 11, + "row": 0, + "col": 0 + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "sizeX": 13, + "sizeY": 5, + "row": 0, + "col": 11 + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "sizeX": 13, + "sizeY": 6, + "row": 5, + "col": 11 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "map": { + "name": "Edit location", + "root": false, + "layouts": { + "main": { + "widgets": { + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "chart": { + "name": "${entityName}", + "root": false, + "layouts": { + "main": { + "widgets": { + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 0, + "col": 6 + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 6, + "col": 6 + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "sizeX": 6, + "sizeY": 6, + "row": 0, + "col": 0 + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "sizeX": 6, + "sizeY": 6, + "row": 6, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "68a058e1-fdda-8482-715b-3ae4a488568e": { + "id": "68a058e1-fdda-8482-715b-3ae4a488568e", + "alias": "Thermostats", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "thermostat", + "deviceNameFilter": "" + } + }, + "12ae98c7-1ea2-52cf-64d5-763e9d993547": { + "id": "12ae98c7-1ea2-52cf-64d5-763e9d993547", + "alias": "Thermostat", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": null, + "defaultStateEntity": null + } + }, + "ce27a9d0-93bf-b7a4-054d-d0369a8cf813": { + "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "alias": "Thermostat-alarm", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "alarm", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1587473857304, + "endTimeMs": 1587560257304 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Thermostats" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json new file mode 100644 index 0000000000..1b321b1e1e --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -0,0 +1,146 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Root Rule Chain", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 2, + "nodes": [ + { + "additionalInfo": { + "layoutX": 824, + "layoutY": 156 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": false, + "configuration": { + "defaultTTL": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 52 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", + "name": "Save Client Attributes", + "debugMode": false, + "configuration": { + "scope": "CLIENT_SCOPE" + } + }, + { + "additionalInfo": { + "layoutX": 347, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", + "name": "Message Type Switch", + "debugMode": false, + "configuration": { + "version": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 266 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log RPC from Device", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 379 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log Other", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 468 + }, + "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", + "name": "RPC Call Request", + "debugMode": false, + "configuration": { + "timeoutInSeconds": 60 + } + }, + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 90 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": false, + "configuration": { + "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" + } + } + ], + "connections": [ + { + "fromIndex": 0, + "toIndex": 6, + "type": "Success" + }, + { + "fromIndex": 2, + "toIndex": 4, + "type": "Other" + }, + { + "fromIndex": 2, + "toIndex": 1, + "type": "Post attributes" + }, + { + "fromIndex": 2, + "toIndex": 0, + "type": "Post telemetry" + }, + { + "fromIndex": 2, + "toIndex": 3, + "type": "RPC Request from Device" + }, + { + "fromIndex": 2, + "toIndex": 5, + "type": "RPC Request to Device" + } + ], + "ruleChainConnections": [ + { + "fromIndex": 6, + "targetRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "83d42540-85fd-11ea-aee2-794850541ced" + }, + "additionalInfo": { + "layoutX": 1088, + "layoutY": 203, + "ruleChainNodeId": "rule-chain-node-9" + }, + "type": "True" + } + ] + } +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json new file mode 100644 index 0000000000..9b7e3401d2 --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -0,0 +1,137 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Thermostat Alarms", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 5, + "nodes": [ + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 67 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;", + "severity": "MAJOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 201 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 131 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;", + "severity": "MINOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 275 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 586, + "layoutY": 148 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", + "name": "Check Alarms", + "debugMode": true, + "configuration": { + "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" + } + }, + { + "additionalInfo": { + "layoutX": 321, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", + "name": "Fetch Configuration", + "debugMode": true, + "configuration": { + "clientAttributeNames": [], + "sharedAttributeNames": [], + "serverAttributeNames": [ + "alarmTemperature", + "thresholdTemperature", + "alarmHumidity", + "thresholdHumidity" + ], + "latestTsKeyNames": [], + "tellFailureIfAbsent": false, + "getLatestValueWithTs": false + } + } + ], + "connections": [ + { + "fromIndex": 4, + "toIndex": 0, + "type": "NewTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 1, + "type": "ClearTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 2, + "type": "NewHumidityAlarm" + }, + { + "fromIndex": 4, + "toIndex": 3, + "type": "ClearHumidityAlarm" + }, + { + "fromIndex": 5, + "toIndex": 4, + "type": "Success" + } + ], + "ruleChainConnections": null + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd907714fa..f024a328c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,24 +25,32 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.Arrays; + @Service @Profile("install") @Slf4j @@ -76,6 +84,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + @Autowired private DeviceCredentialsService deviceCredentialsService; @@ -120,7 +131,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { demoTenant.setRegion("Global"); demoTenant.setTitle("Tenant"); demoTenant = tenantService.saveTenant(demoTenant); - installScripts.createDefaultRuleChains(demoTenant.getId()); + installScripts.loadDemoRuleChains(demoTenant.getId()); createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant"); Customer customerA = new Customer(); @@ -152,6 +163,25 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); + DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + + attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50)))); + + attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30)))); + installScripts.loadDashboards(demoTenant.getId(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 73e1dbf7fd..d61ca861eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,6 +24,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -182,4 +183,54 @@ public class InstallScripts { } + public void loadDemoRuleChains(TenantId tenantId) throws Exception { + Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR); + try { + JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + ruleChain.setTenantId(tenantId); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + + JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile()); + RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class); + + RuleChainId thermostatsRuleChainId = ruleChain.getId(); + rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId)); + rootChain.setTenantId(tenantId); + rootChain = ruleChainService.saveRuleChain(rootChain); + rootChainMetaData.setRuleChainId(rootChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json", e); + throw new RuntimeException("Unable to load dashboard from json", e); + } + + try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { + dirStream.forEach( + path -> { + try { + JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + + ruleChain.setTenantId(tenantId); + if (ruleChain.getName().equals("Root Rule Chain")) { + ruleChain.setRoot(true); + } + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json: [{}]", path.toString()); + throw new RuntimeException("Unable to load dashboard from json", e); + } + } + ); + } + } } From 9b0e92cac30159ea1bb1fc2eeb0fc978f4fa8d72 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:46:48 +0300 Subject: [PATCH 084/102] New Default Dashboard and Rule Chains --- .../server/service/install/DefaultSystemDataLoaderService.java | 2 +- .../org/thingsboard/server/service/install/InstallScripts.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index f024a328c7..b8d5d26d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index d61ca861eb..37f724ad22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From a91a69c57f6737fbdd438e5aeb79eb5bf47e9a13 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 13:55:06 +0300 Subject: [PATCH 085/102] Fixed Docker Compose --- .../service/install/InstallScripts.java | 24 ------------- .../service/script/RemoteJsInvokeService.java | 2 +- docker/docker-compose.yml | 21 ++++------- docker/kafka.env | 2 +- docker/tb-coap-transport.env | 1 + docker/tb-http-transport.env | 1 + docker/tb-mqtt-transport.env | 1 + docker/tb-node.env | 4 +-- docker/tb-node/conf/logback.xml | 4 +-- docker/tb-node/conf/thingsboard.conf | 2 +- docker/tb-transports/coap/conf/logback.xml | 4 +-- .../coap/conf/tb-coap-transport.conf | 2 +- docker/tb-transports/http/conf/logback.xml | 4 +-- .../http/conf/tb-http-transport.conf | 2 +- docker/tb-transports/mqtt/conf/logback.xml | 4 +-- .../mqtt/conf/tb-mqtt-transport.conf | 2 +- k8s/tb-coap-transport-configmap.yml | 6 ++-- k8s/tb-http-transport-configmap.yml | 6 ++-- k8s/tb-mqtt-transport-configmap.yml | 6 ++-- k8s/tb-node-configmap.yml | 6 ++-- k8s/thingsboard.yml | 36 +++++++------------ .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 24 files changed, 53 insertions(+), 93 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 37f724ad22..5181834b80 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -208,29 +208,5 @@ public class InstallScripts { log.error("Unable to load dashboard from json", e); throw new RuntimeException("Unable to load dashboard from json", e); } - - try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { - dirStream.forEach( - path -> { - try { - JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); - RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); - - ruleChain.setTenantId(tenantId); - if (ruleChain.getName().equals("Root Rule Chain")) { - ruleChain.setRoot(true); - } - ruleChain = ruleChainService.saveRuleChain(ruleChain); - - ruleChainMetaData.setRuleChainId(ruleChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); - } catch (Exception e) { - log.error("Unable to load dashboard from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load dashboard from json", e); - } - } - ); - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..fe2e7e8071 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -46,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; @Getter diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c36356b1cb..9774346b3d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -59,8 +59,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb1 - CLUSTER_NODE_ID: tb1 + TB_SERVICE_ID: tb1 env_file: - tb-node.env volumes: @@ -81,8 +80,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb2 - CLUSTER_NODE_ID: tb2 + TB_SERVICE_ID: tb2 env_file: - tb-node.env volumes: @@ -98,8 +96,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport1 - CLUSTER_NODE_ID: tb-mqtt-transport1 + TB_SERVICE_ID: tb-mqtt-transport1 env_file: - tb-mqtt-transport.env volumes: @@ -113,8 +110,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport2 - CLUSTER_NODE_ID: tb-mqtt-transport2 + TB_SERVICE_ID: tb-mqtt-transport2 env_file: - tb-mqtt-transport.env volumes: @@ -128,8 +124,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport1 - CLUSTER_NODE_ID: tb-http-transport1 + TB_SERVICE_ID: tb-http-transport1 env_file: - tb-http-transport.env volumes: @@ -143,8 +138,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport2 - CLUSTER_NODE_ID: tb-http-transport2 + TB_SERVICE_ID: tb-http-transport2 env_file: - tb-http-transport.env volumes: @@ -158,8 +152,7 @@ services: ports: - "5683:5683/udp" environment: - TB_HOST: tb-coap-transport - CLUSTER_NODE_ID: tb-coap-transport + TB_SERVICE_ID: tb-coap-transport env_file: - tb-coap-transport.env volumes: diff --git a/docker/kafka.env b/docker/kafka.env index 485b3c0169..81c01dae93 100644 --- a/docker/kafka.env +++ b/docker/kafka.env @@ -4,7 +4,7 @@ KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE -KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 +KAFKA_CREATE_TOPICS=js_eval.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 KAFKA_AUTO_CREATE_TOPICS_ENABLE=false KAFKA_LOG_RETENTION_BYTES=1073741824 KAFKA_LOG_SEGMENT_BYTES=268435456 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index ed8f78d116..af95bc904d 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -3,4 +3,5 @@ COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index c8363230a2..71bc434241 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -3,4 +3,5 @@ HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index b024c7a1a5..163cfab451 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -3,4 +3,5 @@ MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index 963943dccd..aec55e9205 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -2,9 +2,9 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 -RPC_HOST=${TB_HOST} +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=remote +JS_EVALUATOR=local TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index 4eb8a6a44f..bc694d704d 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-node/conf/thingsboard.conf b/docker/tb-node/conf/thingsboard.conf index 392b3302fd..3f27bf2859 100644 --- a/docker/tb-node/conf/thingsboard.conf +++ b/docker/tb-node/conf/thingsboard.conf @@ -15,7 +15,7 @@ # export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/coap/conf/logback.xml b/docker/tb-transports/coap/conf/logback.xml index f1ad75fd5e..20d4b6bc92 100644 --- a/docker/tb-transports/coap/conf/logback.xml +++ b/docker/tb-transports/coap/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/coap/conf/tb-coap-transport.conf b/docker/tb-transports/coap/conf/tb-coap-transport.conf index 7695e52d44..a64a5f4828 100644 --- a/docker/tb-transports/coap/conf/tb-coap-transport.conf +++ b/docker/tb-transports/coap/conf/tb-coap-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/http/conf/logback.xml b/docker/tb-transports/http/conf/logback.xml index 0aaed3ca3b..67d15ce607 100644 --- a/docker/tb-transports/http/conf/logback.xml +++ b/docker/tb-transports/http/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/http/conf/tb-http-transport.conf b/docker/tb-transports/http/conf/tb-http-transport.conf index b52f5a0f28..67d217e6c3 100644 --- a/docker/tb-transports/http/conf/tb-http-transport.conf +++ b/docker/tb-transports/http/conf/tb-http-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/mqtt/conf/logback.xml b/docker/tb-transports/mqtt/conf/logback.xml index 4a4e19d00e..494759aeb9 100644 --- a/docker/tb-transports/mqtt/conf/logback.xml +++ b/docker/tb-transports/mqtt/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf index 4081d5a194..4fa2550acf 100644 --- a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf +++ b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/k8s/tb-coap-transport-configmap.yml b/k8s/tb-coap-transport-configmap.yml index d2ef3fbb47..55adbcbdec 100644 --- a/k8s/tb-coap-transport-configmap.yml +++ b/k8s/tb-coap-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-coap-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-http-transport-configmap.yml b/k8s/tb-http-transport-configmap.yml index f406fd0d4f..de20a90952 100644 --- a/k8s/tb-http-transport-configmap.yml +++ b/k8s/tb-http-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-http-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-mqtt-transport-configmap.yml b/k8s/tb-mqtt-transport-configmap.yml index cf9b2c655e..9d56f6fa71 100644 --- a/k8s/tb-mqtt-transport-configmap.yml +++ b/k8s/tb-mqtt-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-mqtt-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-node-configmap.yml b/k8s/tb-node-configmap.yml index 7fa9612600..1f6f1abf97 100644 --- a/k8s/tb-node-configmap.yml +++ b/k8s/tb-node-configmap.yml @@ -24,7 +24,7 @@ metadata: data: conf: | export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -37,10 +37,10 @@ data: - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/thingsboard.yml b/k8s/thingsboard.yml index 13fadf7fe2..8f50f34f8e 100644 --- a/k8s/thingsboard.yml +++ b/k8s/thingsboard.yml @@ -247,18 +247,12 @@ spec: - containerPort: 9001 name: rpc env: - - name: RPC_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: ZOOKEEPER_ENABLED value: "true" - name: ZOOKEEPER_URL @@ -334,14 +328,12 @@ spec: - containerPort: 1883 name: mqtt env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: MQTT_BIND_ADDRESS value: "0.0.0.0" - name: MQTT_BIND_PORT @@ -409,14 +401,12 @@ spec: - containerPort: 8080 name: http env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: HTTP_BIND_ADDRESS value: "0.0.0.0" - name: HTTP_BIND_PORT @@ -484,14 +474,12 @@ spec: name: coap protocol: UDP env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: COAP_BIND_ADDRESS value: "0.0.0.0" - name: COAP_BIND_PORT diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..8ff890e476 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -42,7 +42,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..4cd67df020 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -43,7 +43,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..b4af9547c8 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -73,7 +73,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" From b5bedbfdf6e99ee133bad22011af4299a8d38854 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 14:54:25 +0300 Subject: [PATCH 086/102] Refactoring of Docker Compose --- .../demo/rule_chains/root_rule_chain.json | 2 +- .../controller/RuleChainController.java | 14 ++--- .../server/dao/rule/BaseRuleChainService.java | 25 +++++--- docker/README.md | 2 +- docker/docker-compose.cassandra.yml | 18 +++++- docker/docker-compose.postgres.volumes.yml | 10 +++- docker/docker-compose.postgres.yml | 18 +++++- docker/docker-compose.yml | 58 +++++++++++++++++-- docker/docker-install-tb.sh | 2 +- docker/docker-upgrade-tb.sh | 4 +- docker/haproxy/config/haproxy.cfg | 4 +- docker/tb-js-executor.env | 2 +- docker/tb-node.env | 2 +- .../server/msa/ThingsBoardDbInstaller.java | 2 +- 14 files changed, 125 insertions(+), 38 deletions(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 1b321b1e1e..b68d070861 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -3,7 +3,7 @@ "additionalInfo": null, "name": "Root Rule Chain", "firstRuleNodeId": null, - "root": false, + "root": true, "debugMode": false, "configuration": null }, diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 5685947c9a..0bc518e48c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -161,15 +161,15 @@ public class RuleChainController extends BaseController { TenantId tenantId = getCurrentUser().getTenantId(); RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId); if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) { + if (previousRootRuleChain != null) { + previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - - tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), - ComponentLifecycleEvent.UPDATED); - - logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, - null, ActionType.UPDATED, null); + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + ComponentLifecycleEvent.UPDATED); + logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, + null, ActionType.UPDATED, null); + } ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 8ed968a78a..fdd99f97f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -88,26 +88,33 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId()); if (!ruleChain.isRoot()) { RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId()); - if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { - try { + try { + if (previousRootRuleChain == null) { + setRootAndSave(tenantId, ruleChain); + return true; + } else if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); previousRootRuleChain.setRoot(false); ruleChainDao.save(tenantId, previousRootRuleChain); - createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); - ruleChain.setRoot(true); - ruleChainDao.save(tenantId, ruleChain); + setRootAndSave(tenantId, ruleChain); return true; - } catch (ExecutionException | InterruptedException e) { - log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); - throw new RuntimeException(e); } + } catch (ExecutionException | InterruptedException e) { + log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); + throw new RuntimeException(e); } } return false; } + private void setRootAndSave(TenantId tenantId, RuleChain ruleChain) throws ExecutionException, InterruptedException { + createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); + ruleChain.setRoot(true); + ruleChainDao.save(tenantId, ruleChain); + } + @Override public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) { Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id."); diff --git a/docker/README.md b/docker/README.md index ff61c2599b..9436386010 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ In case of any issues you can examine service logs for errors. For example to see ThingsBoard node logs execute the following command: ` -$ docker-compose logs -f tb1 +$ docker-compose logs -f tb-core1 tb-rule-engine1 ` Or use `docker-compose ps` to see the state of all the containers. diff --git a/docker/docker-compose.cassandra.yml b/docker/docker-compose.cassandra.yml index 49de5f396c..1066570140 100644 --- a/docker/docker-compose.cassandra.yml +++ b/docker/docker-compose.cassandra.yml @@ -24,14 +24,28 @@ services: - "9042" volumes: - ./tb-node/cassandra:/var/lib/cassandra - tb1: + tb-core1: env_file: - tb-node.cassandra.env depends_on: - kafka - redis - cassandra - tb2: + tb-core2: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine1: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine2: env_file: - tb-node.cassandra.env depends_on: diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 1331393044..caaacf96e8 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,10 +20,16 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb1: + tb-core1: volumes: - tb-log-volume:/var/log/thingsboard - tb2: + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: volumes: - tb-log-volume:/var/log/thingsboard tb-coap-transport: diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index d47cbd6c37..3d19e5f968 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -27,14 +27,28 @@ services: POSTGRES_PASSWORD: postgres volumes: - ./tb-node/postgres:/var/lib/postgresql/data - tb1: + tb-core1: env_file: - tb-node.postgres.env depends_on: - kafka - redis - postgres - tb2: + tb-core2: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine1: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9774346b3d..783a5ade46 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,7 +48,7 @@ services: - tb-js-executor.env depends_on: - kafka - tb1: + tb-core1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -59,7 +59,8 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb1 + TB_SERVICE_ID: tb-core1 + TB_SERVICE_TYPE: tb-core env_file: - tb-node.env volumes: @@ -69,7 +70,7 @@ services: - kafka - redis - tb-js-executor - tb2: + tb-core2: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -80,7 +81,52 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb2 + TB_SERVICE_ID: tb-core2 + TB_SERVICE_TYPE: tb-core + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine1: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine1 + TB_SERVICE_TYPE: tb-rule-engine + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine2: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine2 + TB_SERVICE_TYPE: tb-rule-engine env_file: - tb-node.env volumes: @@ -195,8 +241,8 @@ services: MQTT_PORT: 1883 FORCE_HTTPS_REDIRECT: "false" links: - - tb1 - - tb2 + - tb-core1 + - tb-core2 - tb-web-ui1 - tb-web-ui2 - tb-mqtt-transport1 diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 1a5d75d64a..cabb3c190d 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,6 +49,6 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 7db7ff6172..e62dbdc438 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,8 +44,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/haproxy/config/haproxy.cfg b/docker/haproxy/config/haproxy.cfg index ba17bd64d9..bc36364024 100644 --- a/docker/haproxy/config/haproxy.cfg +++ b/docker/haproxy/config/haproxy.cfg @@ -111,6 +111,6 @@ backend tb-api-backend balance leastconn option tcp-check option log-health-checks - server tbApi1 tb1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 - server tbApi2 tb2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi1 tb-core1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi2 tb-core2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 http-request set-header X-Forwarded-Port %[dst_port] diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 38f283cd1a..0b64f43b00 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,5 +1,5 @@ -REMOTE_JS_EVAL_REQUEST_TOPIC=js.eval.requests +REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info LOG_FOLDER=logs diff --git a/docker/tb-node.env b/docker/tb-node.env index aec55e9205..542296babe 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,7 +4,7 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=local +JS_EVALUATOR=remote TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 8fa7ae9a3d..d661bb7476 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -92,7 +92,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); dockerCompose.invokeCompose(); } finally { From 2c02406ee5f41b08e655b634dd9871403f67149d Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Fri, 24 Apr 2020 15:44:31 +0300 Subject: [PATCH 087/102] Merge with master. AlarmRepository.findAlarms is failing (#2663) * Merge with master. AlarmRepository.findAlarms is failing * Fix Alarm repository. Add queue type list select. Co-authored-by: Igor Kulikov --- .../data/json/demo/dashboards/gateways.json | 1274 +++++++++++++++++ .../dashboards/rule_engine_statistics.json | 520 +++++++ .../data/upgrade/2.4.3/schema_update_ttl.sql | 25 + .../server/actors/ActorSystemContext.java | 122 +- .../server/actors/app/AppActor.java | 104 +- .../server/actors/device/DeviceActor.java | 13 +- .../device/DeviceActorMessageProcessor.java | 183 +-- .../actors/rpc/BasicRpcSessionListener.java | 83 -- .../server/actors/rpc/RpcManagerActor.java | 230 --- .../server/actors/rpc/RpcSessionActor.java | 135 -- .../server/actors/rpc/RpcSessionTellMsg.java | 27 - .../actors/ruleChain/DefaultTbContext.java | 234 +-- .../RemoteToRuleChainTellNextMsg.java | 48 - .../actors/ruleChain/RuleChainActor.java | 31 +- .../RuleChainActorMessageProcessor.java | 237 +-- .../ruleChain/RuleChainManagerActor.java | 68 +- .../ruleChain/RuleChainToRuleChainMsg.java | 1 - .../actors/ruleChain/RuleNodeActor.java | 4 + .../RuleNodeActorMessageProcessor.java | 8 +- .../RuleNodeToRuleChainTellNextMsg.java | 1 + .../server/actors/service/ActorService.java | 19 +- .../server/actors/service/ComponentActor.java | 6 +- .../actors/service/ContextAwareActor.java | 4 +- .../actors/service/DefaultActorService.java | 204 +-- .../actors/shared/ComponentMsgProcessor.java | 11 +- .../actors/shared/EntityActorsManager.java | 87 -- .../shared/rulechain/RuleChainManager.java | 59 - .../rulechain/SystemRuleChainManager.java | 48 - .../rulechain/TenantRuleChainManager.java | 53 - .../server/actors/stats/StatsActor.java | 7 +- .../server/actors/tenant/TenantActor.java | 161 ++- .../server/config/WebSocketConfiguration.java | 5 +- .../server/controller/AdminController.java | 14 +- .../server/controller/AlarmController.java | 4 +- .../server/controller/AssetController.java | 4 +- .../server/controller/AuditLogController.java | 2 + .../server/controller/AuthController.java | 2 + .../server/controller/BaseController.java | 45 +- .../ComponentDescriptorController.java | 2 + .../server/controller/CustomerController.java | 2 + .../controller/DashboardController.java | 3 +- .../server/controller/DeviceController.java | 16 +- .../controller/EntityRelationController.java | 2 + .../controller/EntityViewController.java | 2 + .../server/controller/EventController.java | 2 + .../server/controller/QueueController.java | 54 + .../server/controller/RpcController.java | 14 +- .../controller/RuleChainController.java | 17 +- .../controller/TelemetryController.java | 20 +- .../server/controller/TenantController.java | 6 +- .../server/controller/UserController.java | 2 + .../controller/WidgetTypeController.java | 2 + .../controller/WidgetsBundleController.java | 3 +- .../controller/plugin/TbWebSocketHandler.java | 9 +- .../install/ThingsboardInstallService.java | 7 +- .../CurrentServerInstanceService.java | 55 - .../cluster/discovery/ServerInstance.java | 47 - .../ConsistentClusterRoutingService.java | 153 -- .../cluster/rpc/ClusterGrpcService.java | 161 --- .../cluster/rpc/ClusterRpcService.java | 42 - .../service/cluster/rpc/GrpcSession.java | 125 -- .../cluster/rpc/GrpcSessionListener.java | 32 - .../service/cluster/rpc/RpcMsgListener.java | 32 - .../encoding/DataDecodingEncodingService.java | 5 - .../service/encoding/ProtoWithFSTService.java | 24 +- .../install/SqlDatabaseUpgradeService.java | 9 + .../CassandraEntitiesToSqlMigrateService.java | 7 +- .../migrate/CassandraToSqlEventTsColumn.java} | 29 +- .../queue/DefaultTbClusterService.java | 204 +++ .../queue/DefaultTbCoreConsumerService.java | 290 ++++ .../DefaultTbRuleEngineConsumerService.java | 262 ++++ .../DefaultTenantRoutingInfoService.java | 47 + .../service/queue/TbClusterService.java | 52 + .../service/queue/TbCoreConsumerService.java | 23 + .../TbCoreConsumerStats.java} | 47 +- .../service/queue/TbMsgPackCallback.java | 48 + .../queue/TbMsgPackProcessingContext.java | 84 ++ .../server/service/queue/TbPackCallback.java | 39 +- .../queue/TbPackProcessingContext.java | 90 ++ .../queue/TbRuleEngineConsumerService.java | 23 + .../queue/TbRuleEngineConsumerStats.java | 132 ++ .../queue/TbTenantRuleEngineStats.java | 92 ++ .../processing/AbstractConsumerService.java | 149 ++ .../AbstractTbRuleEngineSubmitStrategy.java | 71 + .../BatchTbRuleEngineSubmitStrategy.java | 86 ++ .../BurstTbRuleEngineSubmitStrategy.java | 50 + .../processing/IdMsgPair.java} | 20 +- ...lByEntityIdTbRuleEngineSubmitStrategy.java | 108 ++ ...riginatorIdTbRuleEngineSubmitStrategy.java | 44 + ...lByTenantIdTbRuleEngineSubmitStrategy.java | 25 +- .../SequentialTbRuleEngineSubmitStrategy.java | 73 + .../TbRuleEngineProcessingDecision.java | 31 + .../TbRuleEngineProcessingResult.java | 58 + .../TbRuleEngineProcessingStrategy.java | 22 + ...TbRuleEngineProcessingStrategyFactory.java | 140 ++ .../TbRuleEngineSubmitStrategy.java | 39 + .../TbRuleEngineSubmitStrategyFactory.java | 43 + .../service/rpc/DefaultDeviceRpcService.java | 222 --- .../rpc/DefaultTbCoreDeviceRpcService.java | 199 +++ .../rpc/DefaultTbRuleEngineRpcService.java | 177 +++ .../server/service/rpc/DeviceRpcService.java | 41 - .../service/rpc/FromDeviceRpcResponse.java | 1 - .../service/rpc/TbCoreDeviceRpcService.java | 57 + .../rpc/TbRuleEngineDeviceRpcService.java | 32 + .../rpc/ToDeviceRpcRequestActorMsg.java | 5 +- .../script/AbstractJsInvokeService.java | 48 +- .../AbstractNashornJsInvokeService.java | 4 +- .../script/NashornJsInvokeService.java | 10 + .../service/script/RemoteJsInvokeService.java | 99 +- .../script/RemoteJsRequestEncoder.java | 9 +- .../script/RemoteJsResponseDecoder.java | 12 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../auth/rest/RestAuthenticationProvider.java | 1 + .../device/DefaultDeviceAuthService.java | 14 +- .../permission/AccessControlService.java | 2 - .../DefaultAccessControlService.java | 7 - .../DefaultDeviceSessionCacheService.java | 3 +- .../state/DefaultDeviceStateService.java | 439 +++--- .../service/state/DeviceStateService.java | 12 +- .../DefaultRuleEngineStatisticsService.java | 141 ++ .../stats/RuleEngineStatisticsService.java | 23 + .../DefaultSubscriptionManagerService.java | 380 +++++ .../DefaultTbLocalSubscriptionService.java | 228 +++ .../SubscriptionManagerService.java | 38 + .../subscription/TbAttributeSubscription.java | 50 + .../TbAttributeSubscriptionScope.java} | 10 +- .../TbLocalSubscriptionService.java | 36 + .../service/subscription/TbSubscription.java | 52 + .../subscription/TbSubscriptionType.java | 9 +- .../subscription/TbSubscriptionUtils.java | 247 ++++ .../TbTimeseriesSubscription.java | 51 + .../DefaultTelemetrySubscriptionService.java | 675 +-------- .../DefaultTelemetryWebSocketService.java | 74 +- .../TelemetrySubscriptionService.java | 27 +- .../service/telemetry/sub/Subscription.java | 80 -- .../telemetry/sub/SubscriptionUpdate.java | 2 +- .../BaseRuleChainTransactionService.java | 242 ---- .../transaction/TbTransactionTask.java | 44 - .../DefaultTbCoreToTransportService.java | 87 ++ ...e.java => DefaultTransportApiService.java} | 33 +- .../transport/LocalTransportService.java | 227 --- .../RemoteRuleEngineTransportService.java | 251 ---- .../transport/RemoteTransportApiService.java | 111 -- ...der.java => TbCoreToTransportService.java} | 17 +- .../transport/TbCoreTransportApiService.java | 100 ++ .../transport/TransportApiRequestDecoder.java | 31 - .../TransportApiResponseEncoder.java | 30 - .../transport/TransportApiService.java | 8 +- .../msg/TransportToDeviceActorMsgWrapper.java | 5 +- ...rvice.java => AbstractCleanUpService.java} | 40 +- .../ttl/events/EventsCleanUpService.java | 59 + .../AbstractTimeseriesCleanUpService.java | 49 + .../PsqlTimeseriesCleanUpService.java | 2 +- .../TimescaleTimeseriesCleanUpService.java | 2 +- .../service/update/DefaultUpdateService.java | 2 + application/src/main/proto/cluster.proto | 146 -- application/src/main/resources/logback.xml | 3 + .../src/main/resources/thingsboard.yml | 286 ++-- .../controller/AbstractControllerTest.java | 6 +- .../controller/BaseAssetControllerTest.java | 74 +- .../BaseEntityViewControllerTest.java | 22 +- .../controller/ControllerSqlTestSuite.java | 7 + .../server/mqtt/MqttNoSqlTestSuite.java | 7 + .../server/mqtt/MqttSqlTestSuite.java | 7 + ...tractMqttServerSideRpcIntegrationTest.java | 19 +- .../AbstractMqttTelemetryIntegrationTest.java | 24 +- .../server/rules/RuleEngineSqlTestSuite.java | 7 + ...AbstractRuleEngineFlowIntegrationTest.java | 50 +- ...actRuleEngineLifecycleIntegrationTest.java | 27 +- .../ConsistentClusterRoutingServiceTest.java | 105 -- .../ConsistentHashParitionServiceTest.java | 118 ++ .../queue/TbMsgPackProcessingContextTest.java | 66 + .../script/RuleNodeJsScriptEngineTest.java | 15 +- .../script/TestNashornJsInvokeService.java | 5 + .../server/system/SystemNoSqlTestSuite.java | 48 + .../server/system/SystemSqlTestSuite.java | 7 + .../server/dao/alarm/AlarmService.java | 2 +- .../thingsboard/server/dao/util/HsqlDao.java | 4 + .../server/dao/util/NoSqlAnyDao.java | 4 + .../server/dao/util/NoSqlTsDao.java | 4 + .../thingsboard/server/dao/util/PsqlDao.java | 4 + .../server/dao/util/PsqlTsAnyDao.java | 4 + .../thingsboard/server/dao/util/SqlDao.java | 4 +- .../server/dao/util/SqlTsAnyDao.java | 4 + .../thingsboard/server/dao/util/SqlTsDao.java | 4 + .../server/dao/util/TimescaleDBTsDao.java | 4 + .../server/common/data/Tenant.java | 22 + .../server/common/data/alarm/Alarm.java | 1 + .../common/data/{alarm => id}/AlarmId.java | 4 +- .../common/data/id/EntityIdFactory.java | 1 - .../common/data/kv/BaseAttributeKvEntry.java | 4 + .../server/common/msg/MsgType.java | 21 +- .../thingsboard/server/common/msg/TbMsg.java | 103 +- .../msg/plugin/ComponentLifecycleMsg.java | 2 +- .../PartitionChangeMsg.java} | 15 +- .../QueueToRuleEngineMsg.java} | 13 +- .../common/msg/queue/RuleEngineException.java | 38 + .../common/msg/queue/RuleNodeException.java | 58 + .../server/common/msg/queue/ServiceQueue.java | 62 + .../common/msg/queue/ServiceQueueKey.java | 41 +- .../server/common/msg/queue/ServiceType.java | 25 + .../server/common/msg/queue/TbCallback.java | 24 +- .../TbMsgCallback.java} | 25 +- .../common/msg/queue/TopicPartitionInfo.java | 80 ++ common/message/src/main/proto/tbmsg.proto | 9 +- common/queue/pom.xml | 37 + .../server/kafka/TBKafkaAdmin.java | 69 - .../server/kafka/TBKafkaConsumerTemplate.java | 88 -- .../server/kafka/TBKafkaProducerTemplate.java | 131 -- .../server/kafka/TbKafkaResponseTemplate.java | 161 --- .../server/queue/TbQueueAdmin.java | 23 + .../server/queue/TbQueueCallback.java | 23 + .../TbQueueConsumer.java} | 23 +- .../TbQueueHandler.java} | 4 +- .../TbQueueMsg.java} | 9 +- .../server/queue/TbQueueMsgDecoder.java | 24 + .../server/queue/TbQueueMsgHeaders.java | 27 + .../server/queue/TbQueueMsgMetadata.java | 19 + .../server/queue/TbQueueProducer.java | 29 + .../server/queue/TbQueueRequestTemplate.java | 16 +- .../server/queue/TbQueueResponseTemplate.java | 23 + .../azure/servicebus/TbServiceBusAdmin.java | 77 + .../TbServiceBusConsumerTemplate.java | 210 +++ .../TbServiceBusProducerTemplate.java | 110 ++ .../servicebus/TbServiceBusSettings.java | 37 + .../common/AbstractTbQueueTemplate.java} | 24 +- .../common}/AsyncCallbackTemplate.java | 2 +- .../queue/common/DefaultTbQueueMsg.java | 24 +- .../common/DefaultTbQueueMsgHeaders.java | 41 + .../DefaultTbQueueRequestTemplate.java} | 141 +- .../DefaultTbQueueResponseTemplate.java | 163 +++ .../MultipleTbQueueTbMsgCallbackWrapper.java | 47 + .../queue/common/TbProtoJsQueueMsg.java | 43 + .../server/queue/common/TbProtoQueueMsg.java | 55 + .../common/TbQueueTbMsgCallbackWrapper.java | 41 + .../ClusterTopologyChangeEvent.java} | 25 +- .../discovery}/ConsistentHashCircle.java | 16 +- .../ConsistentHashPartitionService.java | 339 +++++ .../DefaultTbServiceInfoProvider.java | 119 ++ .../queue/discovery/DiscoveryService.java | 20 + .../discovery/DummyDiscoveryService.java | 42 +- .../queue/discovery/PartitionChangeEvent.java | 43 + .../queue/discovery/PartitionService.java | 58 + .../discovery/TbServiceInfoProvider.java | 21 +- .../queue/discovery/TenantRoutingInfo.java | 16 +- .../discovery/TenantRoutingInfoService.java | 23 + .../discovery/TopicPartitionInfoKey.java | 44 + .../queue}/discovery/ZkDiscoveryService.java | 227 ++- .../environment/EnvironmentLogService.java | 4 +- .../server/queue/kafka/KafkaTbQueueMsg.java | 54 + .../queue/kafka/KafkaTbQueueMsgMetadata.java | 16 +- .../server/queue/kafka/TbKafkaAdmin.java | 89 ++ .../queue/kafka/TbKafkaConsumerTemplate.java | 159 ++ .../{ => queue}/kafka/TbKafkaDecoder.java | 6 +- .../{ => queue}/kafka/TbKafkaEncoder.java | 2 +- .../queue/kafka/TbKafkaProducerTemplate.java | 112 ++ .../server/queue/kafka/TbKafkaProperty.java | 28 + .../{ => queue}/kafka/TbKafkaSettings.java | 26 +- .../queue/kafka/TbKafkaTopicConfigs.java | 71 + .../server/queue/memory/InMemoryStorage.java | 80 ++ .../queue/memory/InMemoryTbQueueConsumer.java | 98 ++ .../queue/memory/InMemoryTbQueueProducer.java | 58 + .../provider/AwsSqsMonolithQueueFactory.java | 169 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 158 ++ .../AwsSqsTbRuleEngineQueueFactory.java | 135 ++ .../provider/AwsSqsTransportQueueFactory.java | 124 ++ .../InMemoryMonolithQueueFactory.java | 123 ++ .../InMemoryTbTransportQueueFactory.java | 99 ++ .../provider/KafkaMonolithQueueFactory.java | 250 ++++ .../provider/KafkaTbCoreQueueFactory.java | 221 +++ .../KafkaTbRuleEngineQueueFactory.java | 193 +++ .../KafkaTbTransportQueueFactory.java | 138 ++ .../provider/PubSubMonolithQueueFactory.java | 177 +++ .../provider/PubSubTbCoreQueueFactory.java | 149 ++ .../PubSubTbRuleEngineQueueFactory.java | 138 ++ .../provider/PubSubTransportQueueFactory.java | 132 ++ .../RabbitMqMonolithQueueFactory.java | 169 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 158 ++ .../RabbitMqTbRuleEngineQueueFactory.java | 136 ++ .../RabbitMqTransportQueueFactory.java | 129 ++ .../ServiceBusMonolithQueueFactory.java | 142 ++ .../ServiceBusTbCoreQueueFactory.java | 124 ++ .../ServiceBusTbRuleEngineQueueFactory.java | 107 ++ .../ServiceBusTransportQueueFactory.java | 98 ++ .../queue/provider/TbCoreQueueFactory.java | 102 ++ .../provider/TbCoreQueueProducerProvider.java | 74 + .../provider/TbQueueProducerProvider.java | 66 + .../TbRuleEngineProducerProvider.java | 75 + .../provider/TbRuleEngineQueueFactory.java | 89 ++ .../provider/TbTransportQueueFactory.java | 38 + .../TbTransportQueueProducerProvider.java | 68 + .../server/queue/pubsub/TbPubSubAdmin.java | 186 +++ .../pubsub/TbPubSubConsumerTemplate.java | 219 +++ .../pubsub/TbPubSubProducerTemplate.java | 134 ++ .../server/queue/pubsub/TbPubSubSettings.java | 58 + .../pubsub/TbPubSubSubscriptionSettings.java | 71 + .../queue/rabbitmq/TbRabbitMqAdmin.java | 80 ++ .../rabbitmq/TbRabbitMqConsumerTemplate.java | 173 +++ .../rabbitmq/TbRabbitMqProducerTemplate.java | 113 ++ .../rabbitmq/TbRabbitMqQueueArguments.java | 98 ++ .../queue/rabbitmq/TbRabbitMqSettings.java | 65 + .../settings/TbQueueCoreSettings.java} | 17 +- .../TbQueueRemoteJsInvokeSettings.java | 42 + .../settings/TbQueueRuleEngineSettings.java | 49 + .../settings/TbQueueTransportApiSettings.java | 47 + .../TbQueueTransportNotificationSettings.java | 17 +- ...leEngineQueueAckStrategyConfiguration.java | 15 +- .../TbRuleEngineQueueConfiguration.java | 20 +- ...gineQueueSubmitStrategyConfiguration.java} | 10 +- .../queue/sqs/AwsSqsTbQueueMsgMetadata.java | 15 +- .../server/queue/sqs/TbAwsSqsAdmin.java | 76 + .../queue/sqs/TbAwsSqsConsumerTemplate.java | 231 +++ .../queue/sqs/TbAwsSqsProducerTemplate.java | 119 ++ .../queue/sqs/TbAwsSqsQueueAttributes.java | 83 ++ .../sqs/TbAwsSqsSettings.java} | 41 +- .../server/queue/util/TbCoreComponent.java | 26 + .../queue/util/TbRuleEngineComponent.java | 26 + .../queue}/src/main/proto/jsinvoke.proto | 0 common/queue/src/main/proto/queue.proto | 410 ++++++ .../transport/coap/CoapTransportContext.java | 2 +- .../transport/coap/CoapTransportResource.java | 2 + .../transport/coap/CoapTransportService.java | 8 +- .../transport/http/DeviceApiController.java | 4 +- .../transport/http/HttpTransportContext.java | 2 +- .../transport/mqtt/MqttTransportContext.java | 12 +- .../transport/mqtt/MqttTransportHandler.java | 24 +- .../transport/mqtt/MqttTransportService.java | 2 +- .../mqtt/session/GatewayDeviceSessionCtx.java | 2 + .../mqtt/session/GatewaySessionHandler.java | 6 +- common/transport/transport-api/pom.xml | 17 +- .../common/transport/TransportContext.java | 13 +- .../common/transport/TransportService.java | 14 +- .../service/AbstractTransportService.java | 327 ----- .../service/DefaultTransportService.java | 600 ++++++++ .../service/RemoteTransportService.java | 339 ----- .../service/RpcRequestMetadata.java} | 16 +- .../service/ToRuleEngineMsgEncoder.java | 2 +- .../ToTransportMsgResponseDecoder.java | 8 +- .../service/TransportApiRequestEncoder.java | 2 +- .../service/TransportApiResponseDecoder.java | 8 +- .../TransportTenantRoutingInfoService.java | 52 + .../session/DeviceAwareSessionContext.java | 8 +- .../common/transport/util}/JsonUtils.java | 2 +- .../src/main/proto/transport.proto | 244 ---- .../server/dao/alarm/BaseAlarmService.java | 2 +- .../server/dao/entity/BaseEntityService.java | 2 +- .../server/dao/model/BaseEntity.java | 4 +- .../server/dao/model/BaseSqlEntity.java | 5 +- .../server/dao/model/ModelConstants.java | 5 + .../dao/model/sql/AbstractAlarmEntity.java | 4 +- .../dao/model/sql/AbstractAssetEntity.java | 2 +- .../dao/model/sql/AbstractDeviceEntity.java | 6 +- .../model/sql/AbstractEntityViewEntity.java | 7 +- .../dao/model/sql/AdminSettingsEntity.java | 2 +- .../server/dao/model/sql/AlarmEntity.java | 1 + .../server/dao/model/sql/AuditLogEntity.java | 6 +- .../model/sql/ComponentDescriptorEntity.java | 5 +- .../server/dao/model/sql/CustomerEntity.java | 6 +- .../server/dao/model/sql/DashboardEntity.java | 6 +- .../dao/model/sql/DashboardInfoEntity.java | 6 +- .../model/sql/DeviceCredentialsEntity.java | 6 +- .../server/dao/model/sql/EventEntity.java | 20 +- .../server/dao/model/sql/RuleChainEntity.java | 6 +- .../server/dao/model/sql/RuleNodeEntity.java | 6 +- .../server/dao/model/sql/TenantEntity.java | 16 +- .../dao/model/sql/UserCredentialsEntity.java | 6 +- .../server/dao/model/sql/UserEntity.java | 6 +- .../dao/model/sql/WidgetTypeEntity.java | 6 +- .../dao/model/sql/WidgetsBundleEntity.java | 2 +- .../server/dao/sql/JpaAbstractDao.java | 4 +- .../server/dao/sql/alarm/AlarmRepository.java | 18 +- .../server/dao/sql/asset/JpaAssetDao.java | 2 - .../server/dao/sql/audit/JpaAuditLogDao.java | 3 + ...ctComponentDescriptorInsertRepository.java | 8 +- ...qlComponentDescriptorInsertRepository.java | 2 +- .../JpaBaseComponentDescriptorDao.java | 3 - .../dao/sql/customer/JpaCustomerDao.java | 3 - .../sql/dashboard/JpaDashboardInfoDao.java | 3 - .../dao/sql/entityview/JpaEntityViewDao.java | 8 +- .../event/AbstractEventInsertRepository.java | 5 +- .../sql/event/HsqlEventInsertRepository.java | 6 +- .../server/dao/sql/event/JpaBaseEventDao.java | 8 +- .../sql/event/PsqlEventInsertRepository.java | 2 +- .../dao/sql/relation/JpaRelationDao.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 3 - .../server/dao/sql/tenant/JpaTenantDao.java | 5 - .../server/dao/sql/user/JpaUserDao.java | 2 - .../dao/sql/widget/JpaWidgetsBundleDao.java | 2 - ...stractChunkedAggregationTimeseriesDao.java | 4 +- .../timescale/TimescaleTimeseriesDao.java | 4 +- .../server/dao/tenant/TenantServiceImpl.java | 36 +- .../resources/sql/schema-entities-hsql.sql | 6 +- .../main/resources/sql/schema-entities.sql | 30 +- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- .../server/dao/sql/alarm/JpaAlarmDaoTest.java | 2 +- .../test/resources/cassandra-test.properties | 1 + dao/src/test/resources/sql-test.properties | 15 +- .../api/jsInvokeMessageProcessor.js | 5 +- pom.xml | 78 +- .../thingsboard/rest/client/RestClient.java | 23 +- .../api/RuleChainTransactionService.java | 31 - .../api/RuleEngineDeviceRpcRequest.java | 5 +- .../rule/engine/api/RuleEngineRpcService.java | 6 +- .../api/RuleEngineTelemetryService.java | 2 - .../rule/engine/api/TbContext.java | 92 +- .../thingsboard/rule/engine/api/TbNode.java | 4 +- rule-engine/rule-engine-components/pom.xml | 4 +- .../engine/action/TbAbstractAlarmNode.java | 12 +- .../action/TbAbstractCustomerActionNode.java | 6 +- .../action/TbAbstractRelationActionNode.java | 12 +- .../TbCopyAttributesToEntityViewNode.java | 8 +- .../rule/engine/action/TbLogNode.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 7 +- .../TbSaveToCustomCassandraTableNode.java | 6 +- .../rule/engine/aws/sns/TbSnsNode.java | 7 +- .../rule/engine/aws/sqs/TbSqsNode.java | 9 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 6 +- .../rule/engine/delay/TbMsgDelayNode.java | 7 +- .../engine/filter/TbCheckAlarmStatusNode.java | 3 +- .../engine/filter/TbMsgTypeFilterNode.java | 2 +- .../engine/filter/TbMsgTypeSwitchNode.java | 2 +- .../filter/TbOriginatorTypeFilterNode.java | 2 +- .../rule/engine/flow/TbAckNode.java | 57 + .../rule/engine/flow/TbCheckpointNode.java | 59 + .../flow/TbCheckpointNodeConfiguration.java | 21 +- .../rule/engine/gcp/pubsub/TbPubSubNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 30 +- .../rule/engine/kafka/TbKafkaNode.java | 21 +- .../rule/engine/mail/TbSendEmailNode.java | 3 +- .../metadata/TbAbstractGetAttributesNode.java | 3 +- .../TbAbstractGetEntityDetailsNode.java | 2 +- .../engine/metadata/TbEntityGetAttrNode.java | 5 +- .../engine/metadata/TbGetDeviceAttrNode.java | 2 +- .../metadata/TbGetOriginatorFieldsNode.java | 4 +- .../engine/metadata/TbGetTelemetryNode.java | 3 +- .../rule/engine/mqtt/TbMqttNode.java | 4 +- .../rule/engine/rabbitmq/TbRabbitMqNode.java | 4 +- .../rule/engine/rest/TbHttpClient.java | 25 +- .../engine/rest/TbRedisQueueProcessor.java | 125 -- .../rule/engine/rest/TbRestApiCallNode.java | 19 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 11 +- .../rule/engine/rpc/TbSendRPCRequestNode.java | 24 +- .../rpc/TbSendRpcReplyNodeConfiguration.java | 24 +- .../engine/telemetry/TbMsgAttributesNode.java | 4 - .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +- .../telemetry/TelemetryNodeCallback.java | 4 +- .../TbSynchronizationBeginNode.java | 26 +- .../transaction/TbSynchronizationEndNode.java | 13 +- .../transform/TbAbstractTransformNode.java | 2 +- .../EntitiesAlarmOriginatorIdAsyncLoader.java | 2 +- .../util/EntitiesFieldsAsyncLoader.java | 2 +- .../util/EntitiesTenantIdAsyncLoader.java | 2 +- .../rule/engine/action/TbAlarmNodeTest.java | 50 +- .../engine/filter/TbJsFilterNodeTest.java | 18 +- .../engine/filter/TbJsSwitchNodeTest.java | 12 +- .../engine/mail/TbMsgToEmailNodeTest.java | 4 +- .../TbGetCustomerAttributeNodeTest.java | 34 +- .../transform/TbChangeOriginatorNodeTest.java | 7 +- .../transform/TbTransformMsgNodeTest.java | 19 +- tools/pom.xml | 2 +- .../ThingsboardCoapTransportApplication.java | 2 +- .../src/main/resources/tb-coap-transport.yml | 159 +- .../ThingsboardHttpTransportApplication.java | 2 +- .../src/main/resources/tb-http-transport.yml | 159 +- .../ThingsboardMqttTransportApplication.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 177 ++- ui-ngx/src/app/core/http/public-api.ts | 1 + ui-ngx/src/app/core/http/queue.service.ts | 36 + .../home/pages/tenant/tenant.component.html | 10 + .../home/pages/tenant/tenant.component.scss | 22 + .../home/pages/tenant/tenant.component.ts | 7 +- .../queue/queue-type-list.component.html | 43 + .../queue/queue-type-list.component.ts | 177 +++ ui-ngx/src/app/shared/models/public-api.ts | 1 + ui-ngx/src/app/shared/models/queue.models.ts | 22 + ui-ngx/src/app/shared/models/tenant.model.ts | 2 + ui-ngx/src/app/shared/shared.module.ts | 3 + .../assets/locale/locale.constant-en_US.json | 12 +- .../assets/locale/locale.constant-ru_RU.json | 5 + .../assets/locale/locale.constant-uk_UA.json | 5 + ui/package-lock.json | 225 ++- ui/src/app/api/queue.service.js | 40 + ui/src/app/components/queue/index.js | 23 + .../queue/queue-type-list.directive.js | 111 ++ .../app/components/queue/queue-type-list.scss | 15 + .../components/queue/queue-type-list.tpl.html | 43 + ui/src/app/layout/index.js | 2 + ui/src/app/locale/locale.constant-en_US.json | 12 +- ui/src/app/locale/locale.constant-ru_RU.json | 6 + ui/src/app/locale/locale.constant-uk_UA.json | 6 + ui/src/app/tenant/tenant-fieldset.tpl.html | 64 +- 492 files changed, 20113 insertions(+), 7920 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/gateways.json create mode 100644 application/src/main/data/json/demo/dashboards/rule_engine_statistics.json delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java create mode 100644 application/src/main/java/org/thingsboard/server/controller/QueueController.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java rename application/src/main/java/org/thingsboard/server/service/{cluster/routing/ClusterRoutingService.java => install/migrate/CassandraToSqlEventTsColumn.java} (51%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java rename application/src/main/java/org/thingsboard/server/service/{transport/RuleEngineStats.java => queue/TbCoreConsumerStats.java} (63%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java => application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java (50%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java rename application/src/main/java/org/thingsboard/server/service/{transport/ToRuleEngineMsgDecoder.java => queue/processing/IdMsgPair.java} (64%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java => application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java (55%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java rename application/src/main/java/org/thingsboard/server/service/{cluster/discovery/ServerInstanceService.java => subscription/TbAttributeSubscriptionScope.java} (79%) create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java => application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java (82%) create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java rename application/src/main/java/org/thingsboard/server/service/transport/{LocalTransportApiService.java => DefaultTransportApiService.java} (75%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java rename application/src/main/java/org/thingsboard/server/service/transport/{ToTransportMsgEncoder.java => TbCoreToTransportService.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java rename application/src/main/java/org/thingsboard/server/service/ttl/{AbstractTimeseriesCleanUpService.java => AbstractCleanUpService.java} (67%) create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/PsqlTimeseriesCleanUpService.java (96%) rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/TimescaleTimeseriesCleanUpService.java (95%) delete mode 100644 application/src/main/proto/cluster.proto delete mode 100644 application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java create mode 100644 application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java rename common/data/src/main/java/org/thingsboard/server/common/data/{alarm => id}/AlarmId.java (88%) rename common/message/src/main/java/org/thingsboard/server/common/msg/{cluster/ClusterEventMsg.java => queue/PartitionChangeMsg.java} (71%) rename common/message/src/main/java/org/thingsboard/server/common/msg/{system/ServiceToRuleEngineMsg.java => queue/QueueToRuleEngineMsg.java} (73%) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java rename application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java => common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java (50%) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java rename application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java => common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java (66%) rename common/message/src/main/java/org/thingsboard/server/common/msg/{TbMsgTransactionData.java => queue/TbMsgCallback.java} (63%) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaPartitioner.java => queue/TbQueueConsumer.java} (63%) rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaHandler.java => queue/TbQueueHandler.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaRequestIdExtractor.java => queue/TbQueueMsg.java} (81%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java rename application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java => common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java (68%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java rename common/queue/src/main/java/org/thingsboard/server/{kafka/AbstractTbKafkaTemplate.java => queue/common/AbstractTbQueueTemplate.java} (70%) rename common/queue/src/main/java/org/thingsboard/server/{kafka => queue/common}/AsyncCallbackTemplate.java (98%) rename application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java (55%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaRequestTemplate.java => queue/common/DefaultTbQueueRequestTemplate.java} (54%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java rename common/{message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java => queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java} (55%) rename {application/src/main/java/org/thingsboard/server/service/cluster/routing => common/queue/src/main/java/org/thingsboard/server/queue/discovery}/ConsistentHashCircle.java (69%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java rename {application/src/main/java/org/thingsboard/server/service/cluster => common/queue/src/main/java/org/thingsboard/server/queue}/discovery/DummyDiscoveryService.java (58%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java rename application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java => common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java (57%) rename application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java => common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java (71%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java rename {application/src/main/java/org/thingsboard/server/service/cluster => common/queue/src/main/java/org/thingsboard/server/queue}/discovery/ZkDiscoveryService.java (67%) rename {application/src/main/java/org/thingsboard/server/service => common/queue/src/main/java/org/thingsboard/server/queue}/environment/EnvironmentLogService.java (90%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java rename application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java (67%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaDecoder.java (83%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaEncoder.java (94%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaSettings.java (79%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaProperty.java => queue/settings/TbQueueCoreSettings.java} (73%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java rename application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java => common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java (63%) rename application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java (73%) rename application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java (62%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java => queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java} (77%) rename application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java (67%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbNodeIdProvider.java => queue/sqs/TbAwsSqsSettings.java} (50%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java rename {application => common/queue}/src/main/proto/jsinvoke.proto (100%) create mode 100644 common/queue/src/main/proto/queue.proto delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java rename common/{message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java => transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java} (76%) create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java rename {application/src/main/java/org/thingsboard/server/utils => common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util}/JsonUtils.java (97%) delete mode 100644 common/transport/transport-api/src/main/proto/transport.proto delete mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java rename application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java => rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java (57%) delete mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java create mode 100644 ui-ngx/src/app/core/http/queue.service.ts create mode 100644 ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss create mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.html create mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts create mode 100644 ui-ngx/src/app/shared/models/queue.models.ts create mode 100644 ui/src/app/api/queue.service.js create mode 100644 ui/src/app/components/queue/index.js create mode 100644 ui/src/app/components/queue/queue-type-list.directive.js create mode 100644 ui/src/app/components/queue/queue-type-list.scss create mode 100644 ui/src/app/components/queue/queue-type-list.tpl.html diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json new file mode 100644 index 0000000000..f6370696c1 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -0,0 +1,1274 @@ +{ + "title": "Gateways", + "configuration": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "isSystemType": true, + "bundleAlias": "entity_admin_widgets", + "typeAlias": "device_admin_table2", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "entitiesTitle": "List of gateways", + "enableSelectColumnDisplay": true, + "displayEntityLabel": false, + "entityNameColumnTitle": "Gateway Name" + }, + "title": "Devices gateway table", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.3646047595211721 + }, + { + "name": "eventsSent", + "type": "timeseries", + "label": "Sent", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.7235710720767985 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Events", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5085933386303254 + }, + { + "name": "LOGS", + "type": "timeseries", + "label": "Latest log", + "color": "#ffc107", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3504240371585048, + "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" + }, + { + "name": "RemoteLoggingLevel", + "type": "attribute", + "label": "Log level", + "color": "#607d8b", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9785994222542516 + } + ], + "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" + } + ], + "showTitleIcon": true, + "titleIcon": "list", + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "List device", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", + "name": "Add device", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n\n", + "customCss": "", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "78845501-234e-a452-6819-82b5b776e99f", + "name": "Configuration", + "icon": "settings", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "openRightLayout": false, + "setEntityId": true + }, + { + "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", + "name": "Graphs", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname_grafic", + "setEntityId": true + }, + { + "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", + "name": "Edit device", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + }, + { + "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", + "name": "Delete device", + "icon": "delete", + "type": "custom", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" + } + ], + "rowClick": [ + { + "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", + "name": "to_entityname__config", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "setEntityId": true, + "stateEntityParamName": "" + } + ] + } + }, + "id": "94715984-ae74-76e4-20b7-2f956b01ed80" + }, + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 14, + "sizeY": 13, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "LOGS", + "type": "timeseries", + "label": "LOGS", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3496649158709739, + "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 2592000000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Debug events (logs)", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" + }, + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "opcuaEventsProduced", + "type": "timeseries", + "label": "opcuaEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.1477920581839779 + }, + { + "name": "opcuaEventsSent", + "type": "timeseries", + "label": "opcuaEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6500957113784758 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 7, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "Events", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.8156044798125357 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Produced", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.6538259344015449 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 604800000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 6, + "hideEmptyLines": true + }, + "title": "Total Messages", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": false, + "showMax": false, + "showAvg": true, + "showTotal": false + } + }, + "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "eventsSent", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.41414001784591314 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "eventsProduced", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.7819101846284422 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "History information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "bleEventsProduced", + "type": "timeseries", + "label": "bleEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.5625165504526104 + }, + { + "name": "bleEventsSent", + "type": "timeseries", + "label": "bleEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6817950080745288 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 5000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "attributes_card", + "type": "latest", + "title": "New widget", + "sizeX": 7, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "eventsTitle": "Gateway Events Form", + "eventsReg": [ + "EventsProduced", + "EventsSent" + ] + }, + "title": "Gateway events", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "466f046d-6005-a168-b107-60fcb2469cd5" + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "config_form_latest", + "type": "latest", + "title": "New widget", + "sizeX": 10, + "sizeY": 9, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "gatewayTitle": "Gateway configuration (Single device)", + "readOnly": false + }, + "title": "New Gateway configuration (Single device)", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodParams": "{}", + "methodName": "gateway_reboot", + "buttonText": "GATEWAY REBOOT" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodName": "gateway_restart", + "methodParams": "{}", + "buttonText": "gateway restart" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.9743324774725604 + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.5530093635101525 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [], + "useMarkerImageFunction": false, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.Mapnik", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5 + }, + "title": "Gateway Location", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" + } + }, + "states": { + "main_gateway": { + "name": "Gateways", + "root": true, + "layouts": { + "main": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname__config": { + "name": "${entityName} Configuration", + "root": false, + "layouts": { + "main": { + "widgets": { + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "sizeX": 14, + "sizeY": 13, + "row": 0, + "col": 10 + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "sizeX": 10, + "sizeY": 9, + "row": 0, + "col": 0 + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "sizeX": 4, + "sizeY": 2, + "row": 9, + "col": 0 + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "sizeX": 4, + "sizeY": 2, + "row": 11, + "col": 0 + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "sizeX": 6, + "sizeY": 4, + "row": 9, + "col": 4 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname_grafic": { + "name": "${entityName} Details", + "root": false, + "layouts": { + "main": { + "widgets": { + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 4, + "col": 7 + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "sizeX": 7, + "sizeY": 7, + "row": 5, + "col": 0 + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 0, + "col": 7 + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 8, + "col": 7 + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "sizeX": 7, + "sizeY": 5, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "auto 100%", + "autoFillHeight": true, + "mobileAutoFillHeight": true, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "3e0f533a-0db1-3292-184f-06e73535061a": { + "id": "3e0f533a-0db1-3292-184f-06e73535061a", + "alias": "Gateways", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "gateway", + "deviceNameFilter": "" + } + }, + "b2487e75-2fa4-f211-142c-434dfd50c70c": { + "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", + "alias": "Current Gateway", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "settings": { + "stateControllerId": "entity", + "showTitle": true, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true, + "titleColor": "rgba(0,0,0,0.870588)" + } + }, + "name": "Gateways" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json new file mode 100644 index 0000000000..b8231ab880 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -0,0 +1,520 @@ +{ + "title": "Rule Engine Statistics", + "configuration": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "successfulMsgs", + "type": "timeseries", + "label": "${entityName} Successful", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.15490750967648736 + }, + { + "name": "failedMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Failures", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.4186621166514697 + }, + { + "name": "tmpFailed", + "type": "timeseries", + "label": "${entityName} Processing Failures", + "color": "#ffc107", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.49891007198715376 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Queue Stats", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 24, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Chain", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleChainName;" + }, + "_hash": 0.9954481282345906 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Node", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleNodeName;" + }, + "_hash": 0.18580357036589978 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Latest Error", + "color": "#f44336", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).message;" + }, + "_hash": 0.7255162989552142 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Exceptions", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "timeoutMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Timeouts", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.565222981550328 + }, + { + "name": "tmpTimeout", + "type": "timeseries", + "label": "${entityName} Processing Timeouts", + "color": "#9c27b0", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.2679547062508352 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Processing Failures and Timeouts", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" + } + }, + "states": { + "default": { + "name": "Rule Engine Statistics", + "root": true, + "layouts": { + "main": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 0 + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "sizeX": 24, + "sizeY": 5, + "row": 7, + "col": 0 + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 12 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "alias": "TbServiceQueues", + "filter": { + "type": "assetType", + "resolveMultiple": true, + "assetType": "TbServiceQueue", + "assetNameFilter": "" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1586176634823, + "endTimeMs": 1586263034823 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Rule Engine Statistics" +} \ No newline at end of file diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index ff1fb5129b..dda3bd7b1c 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,3 +123,28 @@ BEGIN END LOOP; END $$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 54ac41172c..7e9b68882b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,12 +32,10 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -45,10 +43,11 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -65,24 +64,23 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.RuleEngineTransportService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -106,35 +104,24 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired @Getter @Setter - private ActorService actorService; + private TbServiceInfoProvider serviceInfoProvider; - @Autowired @Getter - private DiscoveryService discoveryService; + @Setter + private ActorService actorService; @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - - @Autowired - @Getter - private ClusterRpcService rpcService; - @Autowired @Getter private DataDecodingEncodingService encodingService; - @Autowired - @Getter - private DeviceAuthService deviceAuthService; - @Autowired @Getter private DeviceService deviceService; @@ -163,6 +150,13 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; + @Autowired + private PartitionService partitionService; + + @Autowired + @Getter + private TbClusterService clusterService; + @Autowired @Getter private TimeseriesService tsService; @@ -195,10 +189,6 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; - @Autowired - @Getter - private DeviceRpcService deviceRpcService; - @Autowired @Getter private JsInvokeService jsSandbox; @@ -211,10 +201,6 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; - @Autowired - @Getter - private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; - @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -231,27 +217,32 @@ public class ActorSystemContext { @Getter private MailService mailService; - @Autowired + //TODO: separate context for TbCore and TbRuleEngine + @Autowired(required = false) @Getter private DeviceStateService deviceStateService; - @Autowired + @Autowired(required = false) @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Lazy - @Autowired + @Autowired(required = false) @Getter - private RuleEngineTransportService ruleEngineTransportService; + private TbCoreToTransportService tbCoreToTransportService; - @Lazy - @Autowired + /** + * The following Service will be null if we operate in tb-core mode + */ + @Autowired(required = false) @Getter - private RuleChainTransactionService ruleChainTransactionService; + private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; - @Value("${cluster.partition_id}") + /** + * The following Service will be null if we operate in tb-rule-engine mode + */ + @Autowired(required = false) @Getter - private long queuePartitionId; + private TbCoreDeviceRpcService tbCoreDeviceRpcService; @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter @@ -269,10 +260,6 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; - @Value("${actors.client_side_rpc.timeout}") - @Getter - private long clientSideRpcTimeout; - @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; @@ -334,11 +321,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Autowired - @Getter - private TbNodeIdProvider nodeIdProvider; - - @Getter @Setter private ActorRef appActor; @@ -365,6 +347,8 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } + + public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -374,7 +358,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -383,7 +367,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -397,8 +381,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); + private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -408,12 +392,21 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(ServerAddress server, String method, String body) { - return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); + private JsonNode toBodyJson(String serviceId, String method, String body) { + return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } - public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, tenantId, entityId); + } + + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, queueName, tenantId, entityId); + } + + + public String getServiceId() { + return serviceInfoProvider.getServiceId(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { @@ -444,7 +437,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -504,7 +497,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("message", "Reached debug mode rate limit!"); if (error != null) { @@ -530,4 +523,7 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } + public void tell(TbActorMsg tbActorMsg, ActorRef sender) { + appActor.tell(tbActorMsg, sender); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..b7782a6a87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,19 +20,13 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; -import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; +import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -42,29 +36,32 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; -public class AppActor extends RuleChainManagerActor { +public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; + private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext, new SystemRuleChainManager(systemContext)); + super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); + this.deletedTenants = new HashSet<>(); } @Override @@ -79,7 +76,7 @@ public class AppActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initRuleChainsAndTenantActors(); + initTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -88,17 +85,14 @@ public class AppActor extends RuleChainManagerActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case SEND_TO_CLUSTER_MSG: - onPossibleClusterMsg((SendToClusterMsg) msg); - break; - case CLUSTER_EVENT_MSG: + case PARTITION_CHANGE_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -106,7 +100,6 @@ public class AppActor extends RuleChainManagerActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -115,16 +108,30 @@ public class AppActor extends RuleChainManagerActor { return true; } - private void initRuleChainsAndTenantActors() { + private void initTenantActors() { log.info("Starting main system actor."); try { - initRuleChains(); - if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - for (Tenant tenant : tenantIterator) { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (isolatedTenantId.isPresent()) { + Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); + if (tenant != null) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); + } else { + log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); + } + } else if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + for (Tenant tenant : tenantIterator) { + if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { + log.debug("[{}] Creating tenant actor", tenant.getId()); + getOrCreateTenantActor(tenant.getId()); + log.debug("[{}] Tenant actor created.", tenant.getId()); + } } } log.info("Main system actor started."); @@ -133,40 +140,33 @@ public class AppActor extends RuleChainManagerActor { } } - private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { - self().tell(msg.getMsg(), ActorRef.noSender()); - } - } - - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { -// this may be a notification about system entities created. -// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); + msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + } else { + msg.getTbMsg().getCallback().onSuccess(); + } } } - @Override protected void broadcast(Object msg) { - super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - target = getEntityActorRef(msg.getEntityId()); + log.warn("Message has system tenant id: {}", msg); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + TenantId tenantId = new TenantId(msg.getEntityId().getId()); + deletedTenants.add(tenantId); + ActorRef tenantActor = tenantActors.get(tenantId); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -183,16 +183,22 @@ public class AppActor extends RuleChainManagerActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + } else { + if (msg instanceof TransportToDeviceActorMsgWrapper) { + ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); + } + } } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.debug("[{}] Creating tenant actor.", tenantId); + log.info("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 547e484652..256ffd0b30 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,10 +22,8 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -48,6 +46,11 @@ public class DeviceActor extends ContextAwareActor { } } + @Override + public void postStop() { + + } + @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { @@ -66,15 +69,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; - case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); - break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; - case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: - processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); - break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 2752b53090..437ed456e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,13 +16,10 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; -import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -38,38 +35,34 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.utils.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -100,9 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; - private final Map toServerRpcPendingMap; - - private final Gson gson = new Gson(); private int rpcSeq = 0; private String deviceName; @@ -117,7 +107,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); - this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -153,7 +142,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (TransportProtos.SessionType.SYNC == value.getType()) { + if (SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -161,7 +150,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -182,16 +171,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - TransportProtos.SessionType sessionType = getSessionType(sessionId); + SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == TransportProtos.SessionType.SYNC) { + if (sessionType == SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -199,7 +188,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == TransportProtos.SessionType.ASYNC) { + if (sessionType == SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -214,7 +203,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -223,8 +212,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); + TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -234,34 +223,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } - if (msg.hasPostAttributes()) { - handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); - reportDeviceActivity = true; - } - if (msg.hasPostTelemetry()) { - handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); - reportDeviceActivity = true; - } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } - if (msg.hasToServerRPCCallRequest()) { - handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); - reportDeviceActivity = true; - } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - if (reportDeviceActivity) { - reportLogicalDeviceActivity(); - } - } - - private void reportLogicalDeviceActivity() { - systemContext.getDeviceStateService().onDeviceActivity(deviceId); + callback.onSuccess(); } private void reportSessionOpen() { @@ -326,67 +297,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { - JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), - TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - - private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { - for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsgMetaData metaData = defaultMetaData.copy(); - metaData.putValue("ts", tsKv.getTs() + ""); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - } - - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { - UUID sessionId = getSessionId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", request.getMethodName()); - json.add("params", JsonUtils.parse(request.getParams())); - - TbMsgMetaData requestMetaData = defaultMetaData.copy(); - requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - - scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); - toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); - } - - private TransportProtos.SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; - } - - void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); - if (data != null) { - log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(msg.getId()).setError("timeout").build() - , data.getSessionId(), data.getNodeId()); - } - } - - void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { - int requestId = msg.getMsg().getRequestId(); - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() - , data.getSessionId(), data.getNodeId()); - } else { - log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); - } - } - - private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); + private SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -434,7 +346,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -449,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -470,7 +382,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -494,10 +406,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -511,10 +424,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -525,6 +438,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } @@ -536,11 +450,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -552,35 +466,35 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } - private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } @@ -632,9 +546,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -643,11 +557,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); - TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); + SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -665,27 +579,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() + SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } @@ -706,4 +620,5 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java deleted file mode 100644 index 3832d6eb94..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionTellMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,18 +17,13 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -40,19 +35,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -67,11 +58,13 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -79,6 +72,7 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ +@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -91,6 +85,11 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } + @Override + public void tellSuccess(TbMsg msg) { + tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); + } + @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); @@ -101,16 +100,11 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } - @Override - public void tellNext(TbMsg msg, String relationType, Throwable th) { - tellNext(msg, Collections.singleton(relationType), th); - } - private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } @Override @@ -119,10 +113,94 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } + @Override + public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + @Override + public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { + RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); + RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); + tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); + TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)) + .addAllRelationTypes(relationTypes); + if (failureMessage != null) { + msg.setFailureMessage(failureMessage); + } + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } + tbMsg.getCallback().onSuccess(); + } + @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -134,66 +212,48 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), + msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } - @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); + return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); - } - - @Override - public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); + return TbMsg.transformMsg(origMsg, type, originator, metaData, data); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process customer created msg: " + e); - } + return entityCreatedMsg(customer, customer.getId(), ruleNodeId); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process device created msg: " + e); - } + return entityCreatedMsg(device, device.getId(), ruleNodeId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process asset created msg: " + e); - } + return entityCreatedMsg(asset, asset.getId(), ruleNodeId); } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { + return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + } + + public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process alarm created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); } } - @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -251,8 +311,8 @@ class DefaultTbContext implements TbContext { } @Override - public String getNodeId() { - return mainCtx.getNodeIdProvider().getNodeId(); + public String getServiceId() { + return mainCtx.getServiceInfoProvider().getServiceId(); } @Override @@ -320,11 +380,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } - @Override - public RuleChainTransactionService getRuleChainTransactionService() { - return mainCtx.getRuleChainTransactionService(); - } - @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); @@ -341,35 +396,7 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return new RuleEngineRpcService() { - @Override - public void sendRpcReply(DeviceId deviceId, int requestId, String body) { - mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); - } - - @Override - public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - }; + return mainCtx.getTbRuleEngineDeviceRpcService(); } @Override @@ -387,10 +414,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } - @Override - public String getServerAddress() { - return mainCtx.getServerAddress(); - } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -398,4 +421,29 @@ class DefaultTbContext implements TbContext { return metaData; } + private class SimpleTbQueueCallback implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } else { + log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java deleted file mode 100644 index d062c5dc49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.ruleChain; - -import lombok.Data; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.aware.TenantAwareMsg; - -import java.io.Serializable; - -/** - * Created by ashvayka on 19.03.18. - */ -@Data -final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { - - private static final long serialVersionUID = 2459605482321657447L; - private final TenantId tenantId; - private final RuleChainId ruleChainId; - - public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { - super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); - this.tenantId = tenantId; - this.ruleChainId = ruleChainId; - } - - @Override - public MsgType getMsgType() { - return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 72faaef533..027f4aac9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.actors.ruleChain; -import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { - super(systemContext, tenantId, ruleChainId); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { + super(systemContext, tenantId, ruleChain.getId()); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, context().parent(), context().self())); } @@ -43,20 +43,17 @@ public class RuleChainActor extends ComponentActor { - private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; + private final TbClusterService clusterService; + private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - private String ruleChainName; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChainId); + super(systemContext, tenantId, ruleChain.getId()); + this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.ruleChainName = ruleChainId.toString(); + this.clusterService = systemContext.getClusterService(); } @Override @@ -92,7 +96,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -152,8 +155,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { @@ -192,100 +195,123 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); + onTellNext(envelope.getMsg(), envelope.getOriginator(), envelope.getRelationTypes(), envelope.getFailureMessage()); + } - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { - onLocalTellNext(envelope); + private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set relationTypes, String failureMessage) { + try { + checkActive(); + EntityId entityId = msg.getOriginator(); + TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(relationTypes, r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + if (relationTypes.contains(TbRelationTypes.FAILURE)) { + RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); + if (ruleNodeCtx != null) { + msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); + } else { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); + msg.getCallback().onFailure(new RuleEngineException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); + } + } else { + msg.getCallback().onSuccess(); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(tpi, msg, relation.getOut(), relation.getType()); + } + } else { + MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + putToQueue(tpi, msg, callbackWrapper, target); + } + } + } catch (Exception e) { + msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage())); } } - private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); - envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { + switch (target.getEntityType()) { + case RULE_NODE: + putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); + break; + case RULE_CHAIN: + putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); + break; + } } - private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - RuleNodeId originatorNodeId = envelope.getOriginator(); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(msg, relation.getOut(), relation.getType()); + private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { + if (tpi.isMyPartition()) { + switch (target.getEntityType()) { + case RULE_NODE: + pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); + break; + case RULE_CHAIN: + parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); + break; } } else { - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - switch (target.getEntityType()) { - case RULE_NODE: - enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); - break; - case RULE_CHAIN: - enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); - break; - } - } - //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } + putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); } } + private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { + ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(newMsg)) + .build(); + clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); + } + private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -298,38 +324,13 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; - public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { + public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.ruleChainManager = ruleChainManager; + this.tenantId = tenantId; + this.actors = HashBiMap.create(); this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - ruleChainManager.init(this.context()); + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + RuleChainId ruleChainId = ruleChain.getId(); + log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); + visit(ruleChain, actorRef); + log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); + } + } + + protected void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { + return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { + return actors.computeIfAbsent(ruleChainId, eId -> { + RuleChain ruleChain = provider.apply(eId); + return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) + .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); + }); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - switch (entityId.getEntityType()) { - case RULE_CHAIN: - target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); - break; + if (entityId.getEntityType() == EntityType.RULE_CHAIN) { + target = getOrCreateActor(this.context(), (RuleChainId) entityId); } return target; } protected void broadcast(Object msg) { - ruleChainManager.broadcast(msg); + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } + + public ActorRef get(RuleChainId id) { + return actors.get(id); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java index e081768d93..0122471c52 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java @@ -32,7 +32,6 @@ public final class RuleChainToRuleChainMsg implements TbActorMsg, RuleChainAware private final RuleChainId source; private final TbMsg msg; private final String fromRelationType; - private final boolean enqueue; @Override public RuleChainId getRuleChainId() { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 65227818e8..abb98468bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -53,6 +54,9 @@ public class RuleNodeActor extends ComponentActor relationTypes; private final TbMsg msg; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 3032b1dfe0..f6403c5d56 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -15,24 +15,7 @@ */ package org.thingsboard.server.actors.service; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; +public interface ActorService { -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { - - void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - - void onMsg(SendToClusterMsg msg); - - void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); - - void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java index 3260c7ee31..2a92caf4f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.actors.stats.StatsPersistMsg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; /** @@ -115,9 +115,9 @@ public abstract class ComponentActor status = system.terminate(); @@ -132,157 +97,4 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onMsg(SendToClusterMsg msg) { - appActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - @Override - public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { - DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - @Override - public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { - log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); - DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - public void broadcast(ToAllNodesMsg msg) { - actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); - appActor.tell(msg, ActorRef.noSender()); - } - - private void broadcast(ClusterEventMsg msg) { - this.appActor.tell(msg, ActorRef.noSender()); - this.rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); - private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); - - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int sent = sentClusterMsgs.getAndSet(0); - int received = receivedClusterMsgs.getAndSet(0); - if (sent > 0 || received > 0) { - log.info("Cluster msgs sent [{}] received [{}]", sent, received); - } - } - } - - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onDeviceAdded(Device device) { - deviceStateService.onDeviceAdded(device); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 2fdc918f8e..1443c61e16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,20 +16,13 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; -import akka.event.LoggingAdapter; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; - -import javax.annotation.Nullable; -import java.util.function.Consumer; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -50,7 +43,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; + public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java deleted file mode 100644 index b824cf61ff..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared; - -import akka.actor.ActorContext; -import akka.actor.ActorRef; -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.common.data.SearchTextBased; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageDataIterable; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class EntityActorsManager> { - - protected final ActorSystemContext systemContext; - protected final BiMap actors; - - public EntityActorsManager(ActorSystemContext systemContext) { - this.systemContext = systemContext; - this.actors = HashBiMap.create(); - } - - protected abstract TenantId getTenantId(); - - protected abstract String getDispatcherName(); - - protected abstract Creator creator(T entityId); - - protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); - - public void init(ActorContext context) { - for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { - T entityId = (T) entity.getId(); - log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(context, entityId); - visit(entity, actorRef); - log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); - } - } - - public void visit(M entity, ActorRef actorRef) { - } - - public ActorRef getOrCreateActor(ActorContext context, T entityId) { - return actors.computeIfAbsent(entityId, eId -> - context.actorOf(Props.create(creator(eId)) - .withDispatcher(getDispatcherName()), eId.toString())); - } - - public void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); - } - - public void remove(T id) { - actors.remove(id); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java deleted file mode 100644 index ef2b4282d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorRef; -import akka.japi.Creator; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.shared.EntityActorsManager; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.rule.RuleChainService; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class RuleChainManager extends EntityActorsManager { - - protected final RuleChainService service; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; - - public RuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - this.service = systemContext.getRuleChainService(); - } - - @Override - public Creator creator(RuleChainId entityId) { - return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); - } - - @Override - public void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java deleted file mode 100644 index 10005b040b..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.model.ModelConstants; - -import java.util.Collections; - -public class SystemRuleChainManager extends RuleChainManager { - - public SystemRuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> new PageData<>(); - } - - @Override - protected TenantId getTenantId() { - return ModelConstants.SYSTEM_TENANT; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java deleted file mode 100644 index cfe862af49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorContext; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; - -public class TenantRuleChainManager extends RuleChainManager { - - private final TenantId tenantId; - - public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext); - this.tenantId = tenantId; - } - - @Override - public void init(ActorContext context) { - super.init(context); - } - - @Override - protected TenantId getTenantId() { - return tenantId; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> service.findTenantRuleChains(tenantId, link); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..717dcecc19 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { @@ -58,12 +57,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 4aacfb55bf..3cee82127c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,41 +22,43 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.device.DeviceActorCreator; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { - private final TenantId tenantId; private final BiMap deviceActors; + private boolean isRuleEngineForCurrentTenant; + private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); - this.tenantId = tenantId; + super(systemContext, tenantId); this.deviceActors = HashBiMap.create(); } @@ -65,12 +67,37 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } + boolean cantFindTenant = false; + @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - initRuleChains(); - log.info("[{}] Tenant actor started.", tenantId); + Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; + } + } + log.info("[{}] Tenant actor started.", tenantId); + } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -83,18 +110,36 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { + if (cantFindTenant) { + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; + } switch (msg.getMsgType()) { - case CLUSTER_EVENT_MSG: - broadcast(msg); + case PARTITION_CHANGE_MSG: + PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; + ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType(); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + //To Rule Chain Actors + broadcast(msg); + } else if (ServiceType.TB_CORE.equals(serviceType)) { + //To Device Actors + List repartitionedDevices = + deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); + for (DeviceId deviceId : repartitionedDevices) { + ActorRef deviceActor = deviceActors.remove(deviceId); + context().stop(deviceActor); + } + } break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); - break; - case DEVICE_ACTOR_TO_RULE_ENGINE_MSG: - onDeviceActorToRuleEngineMsg((DeviceActorToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -105,7 +150,6 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -114,41 +158,59 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); - } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); - } + private boolean isMyPartition(DeviceId deviceId) { + return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); } - private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + if (!isRuleEngineForCurrentTenant) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + return; + } + TbMsg tbMsg = msg.getTbMsg(); + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg, self()); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); + ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); + if (ruleChainActor != null) { + ruleChainActor.tell(msg, self()); + } else { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { + if (!isCore) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - ruleChainManager.visit(ruleChain, target); + if (isRuleEngineForCurrentTenant) { + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + visit(ruleChain, target); + } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -172,7 +234,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.warn("[{}] Removed actor was not found in the device map!"); + log.debug("Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); @@ -195,15 +257,12 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { - @Override - public SupervisorStrategy.Directive apply(Throwable t) { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); - } + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); } }); diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 5e5bd94afa..e57695da69 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,11 +32,13 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration +@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { @@ -62,7 +64,8 @@ public class WebSocketConfiguration implements WebSocketConfigurer { SecurityUser user = null; try { user = getCurrentUser(); - } catch (ThingsboardException ex) {} + } catch (ThingsboardException ex) { + } if (user == null) { response.setStatusCode(HttpStatus.UNAUTHORIZED); return false; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index b7efc180a5..8873d82366 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,23 +25,25 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.common.data.security.model.SecuritySettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; -import org.thingsboard.server.common.data.UpdateMessage; @RestController +@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { @Autowired private MailService mailService; - + @Autowired private AdminSettingsService adminSettingsService; @@ -65,7 +67,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/settings", method = RequestMethod.POST) - @ResponseBody + @ResponseBody public AdminSettings saveAdminSettings(@RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -111,8 +113,8 @@ public class AdminController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); adminSettings = checkNotNull(adminSettings); if (adminSettings.getKey().equals("mail")) { - String email = getCurrentUser().getEmail(); - mailService.sendTestMail(adminSettings.getJsonValue(), email); + String email = getCurrentUser().getEmail(); + mailService.sendTestMail(adminSettings.getJsonValue(), email); } } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 9999ab6709..85ee63b3b1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -41,12 +41,14 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.UUID; @RestController +@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 87a5ab3fc4..6963a35d6f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -33,16 +33,15 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -52,6 +51,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 8612d9b027..7eb74fef90 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -39,6 +40,7 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 44449eb3d6..42da043a91 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -56,6 +57,7 @@ import java.net.URI; import java.net.URISyntaxException; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 02e0cef53c..36bc2b104c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -27,16 +26,27 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -64,8 +74,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -86,7 +94,11 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -105,6 +117,7 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j +@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -155,7 +168,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected ActorService actorService; + protected TbClusterService tbClusterService; @Autowired protected RelationService relationService; @@ -178,6 +191,12 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; + @Autowired + protected PartitionService partitionService; + + @Autowired + protected TbQueueProducerProvider producerProvider; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -694,10 +713,14 @@ public abstract class BaseController { } } } - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg))); + TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + TenantId tenantId = user.getTenantId(); + if (tenantId.isNullUid()) { + if (entity instanceof HasTenantId) { + tenantId = ((HasTenantId) entity).getTenantId(); + } + } + tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index ef9508ad7c..d75da8b34c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,12 +25,14 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 65c031172d..5d7e662a72 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,10 +36,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index a0b3904d7a..1b972a5319 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -47,6 +47,7 @@ import java.util.HashSet; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index c2e82f897e..abf00e2bca 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -31,6 +31,8 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; @@ -51,6 +53,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -62,6 +65,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { @@ -109,12 +113,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - actorService - .onDeviceNameOrTypeUpdate( - savedDevice.getTenantId(), - savedDevice.getId(), - savedDevice.getName(), - savedDevice.getType()); + tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -267,7 +267,9 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); + + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); + logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 8058cca685..88aa9ce850 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @@ -40,6 +41,7 @@ import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 67c8ea0688..1611e71c27 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -61,6 +62,7 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index cba2b2bcc0..cf868b29a2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController +@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java new file mode 100644 index 0000000000..a0b46f9166 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +public class QueueController extends BaseController { + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) + @ResponseBody + public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { + checkParameter("serviceType", serviceType); + try { + ServiceType type = ServiceType.valueOf(serviceType); + switch (type) { + case TB_RULE_ENGINE: + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); + default: + return Collections.emptyList(); + } + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index a8298ce82b..b88c3f3695 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -31,7 +31,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -43,27 +42,25 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { @@ -72,7 +69,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private DeviceRpcService deviceRpcService; + private TbCoreDeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -91,7 +88,6 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } - private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); @@ -116,7 +112,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index cc6d4447e9..a97b976bb7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -52,8 +52,10 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; @Slf4j @RestController +@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { @@ -130,7 +133,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -161,7 +164,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -169,7 +172,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -203,7 +206,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -255,9 +258,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, @@ -318,7 +321,7 @@ public class RuleChainController extends BaseController { ScriptEngine engine = null; try { engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); - TbMsg inMsg = new TbMsg(UUIDs.timeBased(), msgType, null, new TbMsgMetaData(metadata), data, null, null, 0L); + TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": output = msgToOutput(engine.executeUpdate(inMsg)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index f607d57cf6..bb866e3e9b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -69,9 +69,9 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -99,6 +99,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { @@ -343,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -353,19 +354,18 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify), null); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -397,12 +397,6 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { - DeviceId deviceId = new DeviceId(entityId.getId()); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index f9781269d7..18d3e80222 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,11 +34,13 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { @@ -74,7 +76,6 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); - tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -94,8 +95,7 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - - actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index c93d672e07..7f190f93cd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -55,6 +56,7 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController +@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index ce3e65c0a1..debe49b018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,12 +31,14 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index b13b3aea54..c0cf672c6f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,13 +32,14 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index f2862c5ab6..6e33133522 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -52,6 +53,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service +@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { @@ -63,7 +65,6 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.send_timeout:5000}") private long sendTimeout; - @Value("${server.ws.limits.max_sessions_per_tenant:0}") private int maxSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer:0}") @@ -91,10 +92,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -138,7 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index e0daf43bf7..7137155ad5 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; @@ -152,6 +152,11 @@ public class ThingsboardInstallService { log.info("Updating system data..."); + systemDataLoaderService.updateSystemWidgets(); + break; + case "2.5.0": + log.info("Upgrading ThingsBoard from version 2.5 to 3.0 ..."); + log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java deleted file mode 100644 index 23b050634c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { - - @Autowired - private ServerInstanceService instanceService; - - @Autowired - private DataDecodingEncodingService encodingService; - - private RpcMsgListener listener; - - private Server server; - - private ServerInstance instance; - - private ConcurrentMap>> pendingSessionMap = - new ConcurrentHashMap<>(); - - public void init(RpcMsgListener listener) { - this.listener = listener; - log.info("Initializing RPC service!"); - instance = instanceService.getSelf(); - server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); - log.info("Going to start RPC server using port: {}", instance.getPort()); - try { - server.start(); - } catch (IOException e) { - log.error("Failed to start RPC server!", e); - throw new RuntimeException("Failed to start RPC server!"); - } - log.info("RPC service initialized!"); - } - - @Override - public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { - BlockingQueue> queue = pendingSessionMap.remove(msgUid); - if (queue != null) { - try { - queue.put(inputStream); - } catch (InterruptedException e) { - log.warn("Failed to report created session!"); - Thread.currentThread().interrupt(); - } - } else { - log.warn("Failed to lookup pending session!"); - } - } - - @Override - public StreamObserver handleMsgs( - StreamObserver responseObserver) { - log.info("Processing new session."); - return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); - } - - - @PreDestroy - public void stop() { - if (server != null) { - log.info("Going to onStop RPC server"); - server.shutdownNow(); - try { - server.awaitTermination(); - log.info("RPC server stopped!"); - } catch (InterruptedException e) { - log.warn("Failed to onStop RPC server!"); - Thread.currentThread().interrupt(); - } - } - } - - - @Override - public void broadcast(RpcBroadcastMsg msg) { - listener.onBroadcastMsg(msg); - } - - private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { - BlockingQueue> queue = new ArrayBlockingQueue<>(1); - pendingSessionMap.put(msg.getMsgUid(), queue); - listener.onRpcSessionCreateRequestMsg(msg); - try { - StreamObserver observer = queue.take(); - log.info("Processed new session."); - return observer; - } catch (Exception e) { - log.info("Failed to process session.", e); - throw new RuntimeException(e); - } - } - - @Override - public void tell(ClusterAPIProtos.ClusterMessage message) { - listener.onSendMsg(message); - } - - @Override - public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { - listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); - } - - @Override - public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { - ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(msgType) - .setPayload(ByteString.copyFrom(data)).build(); - listener.onSendMsg(msg); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java deleted file mode 100644 index 637924fca9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.stub.StreamObserver; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRpcService { - - void init(RpcMsgListener listener); - - void broadcast(RpcBroadcastMsg msg); - - void onSessionCreated(UUID msgUid, StreamObserver inputStream); - - void tell(ClusterAPIProtos.ClusterMessage message); - - void tell(ServerAddress serverAddress, TbActorMsg actorMsg); - - void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java deleted file mode 100644 index aff0bc4a41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.io.Closeable; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -@Slf4j -public final class GrpcSession implements Closeable { - private final UUID sessionId; - private final boolean client; - private final GrpcSessionListener listener; - private final ManagedChannel channel; - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private ServerAddress remoteServer; - - public GrpcSession(GrpcSessionListener listener) { - this(null, listener, null); - } - - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { - this.sessionId = UUID.randomUUID(); - this.listener = listener; - if (remoteServer != null) { - this.client = true; - this.connected = true; - this.remoteServer = remoteServer; - } else { - this.client = false; - } - this.channel = channel; - } - - public void initInputStream() { - this.inputStream = new StreamObserver() { - @Override - public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { - if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { - connected = true; - ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); - remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); - listener.onConnected(GrpcSession.this); - } - if (connected) { - listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); - } - } - - @Override - public void onError(Throwable t) { - listener.onError(GrpcSession.this, t); - } - - @Override - public void onCompleted() { - outputStream.onCompleted(); - listener.onDisconnected(GrpcSession.this); - } - }; - } - - public void initOutputStream() { - if (client) { - listener.onConnected(GrpcSession.this); - } - } - - public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (connected) { - try { - outputStream.onNext(msg); - } catch (Throwable t) { - try { - outputStream.onError(t); - } catch (Throwable t2) { - } - listener.onError(GrpcSession.this, t); - } - } else { - log.warn("[{}] Failed to send message due to closed session!", sessionId); - } - } - - @Override - public void close() { - connected = false; - try { - outputStream.onCompleted(); - } catch (IllegalStateException e) { - log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); - } - if (channel != null) { - channel.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java deleted file mode 100644 index a6ecf967d6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -public interface GrpcSessionListener { - - void onConnected(GrpcSession session); - - void onDisconnected(GrpcSession session); - - void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); - - void onError(GrpcSession session, Throwable t); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java deleted file mode 100644 index 2ff37b39e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ - -public interface RpcMsgListener { - void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); - void onSendMsg(ClusterAPIProtos.ClusterMessage msg); - void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); - void onBroadcastMsg(RpcBroadcastMsg msg); -} diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index a5d3ab465e..4a781b8673 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -27,8 +25,5 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); - ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 45bb9f78fd..8d89059488 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,25 +15,19 @@ */ package org.thingsboard.server.service.encoding; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - - @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + @Override public Optional decode(byte[] byteArray) { try { @@ -42,7 +36,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -51,18 +45,4 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } - @Override - public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg) { - return ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .setPayload(ByteString.copyFrom(encode(msg))).build(); - - } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..87164737c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,15 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + } catch (Exception e) { + } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index 08a5fdd3e9..e7f21a18d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -203,7 +203,8 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("entity_type"), stringColumn("event_type"), stringColumn("event_uid"), - stringColumn("body")), + stringColumn("body"), + new CassandraToSqlEventTsColumn()), new CassandraToSqlTable("relation", idColumn("from_id"), stringColumn("from_type"), @@ -245,7 +246,9 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("zip"), stringColumn("phone"), stringColumn("email"), - stringColumn("additional_info")), + stringColumn("additional_info"), + booleanColumn("isolated_tb_core"), + booleanColumn("isolated_tb_rule_engine")), new CassandraToSqlTable("user_credentials", idColumn("id"), idColumn("user_id"), diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java similarity index 51% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java rename to application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java index e0ed64fdbd..8e369c66c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.service.install.migrate; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; +import com.datastax.driver.core.Row; -import java.util.Optional; import java.util.UUID; -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; + +public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { - ServerAddress getCurrentServer(); + CassandraToSqlEventTsColumn() { + super("id", "ts", CassandraToSqlColumnType.BIGINT, null); + } - Optional resolveById(EntityId entityId); + @Override + public String getColumnValue(Row row) { + UUID id = row.getUUID(getIndex()); + long ts = getTs(id); + return ts + ""; + } + private long getTs(UUID uuid) { + return (uuid.timestamp() - EPOCH_DIFF) / 10000; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java new file mode 100644 index 0000000000..926b08f38b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -0,0 +1,204 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@Slf4j +public class DefaultTbClusterService implements TbClusterService { + + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger toCoreMsgs = new AtomicInteger(0); + private final AtomicInteger toCoreNfs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); + private final AtomicInteger toTransportNfs = new AtomicInteger(0); + + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final DataDecodingEncodingService encodingService; + + public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.encodingService = encodingService; + } + + @Override + public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + byte[] msgBytes = encodingService.encode(msg); + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toCoreNfs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toRuleEngineNfs.incrementAndGet(); + } + + @Override + public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); + toTransportNfs.incrementAndGet(); + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + private void broadcast(ComponentLifecycleMsg msg) { + byte[] msgBytes = encodingService.encode(msg); + TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); + for (String serviceId : tbCoreServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfs.incrementAndGet(); + } + // No need to push notifications twice + tbRuleEngineServices.removeAll(tbCoreServices); + } + for (String serviceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + } + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int toCoreMsgCnt = toCoreMsgs.getAndSet(0); + int toCoreNfsCnt = toCoreNfs.getAndSet(0); + int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); + int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); + int toTransportNfsCnt = toTransportNfs.getAndSet(0); + if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { + log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", + toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java new file mode 100644 index 0000000000..a5abd77ad5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -0,0 +1,290 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@Slf4j +public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { + + @Value("${queue.core.poll-interval}") + private long pollDuration; + @Value("${queue.core.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.core.stats.enabled:false}") + private boolean statsEnabled; + + private final TbQueueConsumer> mainConsumer; + private final DeviceStateService stateService; + private final TbLocalSubscriptionService localSubscriptionService; + private final SubscriptionManagerService subscriptionManagerService; + private final TbCoreDeviceRpcService tbCoreDeviceRpcService; + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, + DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService) { + super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.stateService = stateService; + this.localSubscriptionService = localSubscriptionService; + this.subscriptionManagerService = subscriptionManagerService; + this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; + } + + @PostConstruct + public void init() { + super.init("tb-core-consumer", "tb-core-notifications-consumer"); + } + + @PreDestroy + public void destroy(){ + super.destroy(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumersExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = mainConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCoreMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + mainConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Core Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_CORE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCoreNotificationMsg toCoreNotification = msg.getValue(); + if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreNotification.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); + } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + if (statsEnabled) { + stats.log(toCoreNotification); + } + } + + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); + callback.onSuccess(); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbCallback callback) { + if (msg.hasSubUpdate()) { + localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { + if (msg.hasAttributeSub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); + } else if (msg.hasTelemetrySub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); + } else if (msg.hasSubClose()) { + TbSubscriptionCloseProto closeProto = msg.getSubClose(); + subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); + } else if (msg.hasTsUpdate()) { + TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + subscriptionManagerService.onTimeSeriesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); + } else if (msg.hasAttrUpdate()) { + TbAttributeUpdateProto proto = msg.getAttrUpdate(); + subscriptionManagerService.onAttributesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); + } else { + throwNotHandled(msg, callback); + } + if (statsEnabled) { + stats.log(msg); + } + } + + private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + @Override + protected void stopMainConsumers() { + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java new file mode 100644 index 0000000000..0c48c54412 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -0,0 +1,262 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import com.google.protobuf.ProtocolStringList; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final RuleEngineStatisticsService statisticsService; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); + private ExecutorService submitExecutor; + + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, + TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + TbQueueRuleEngineSettings ruleEngineSettings, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, + ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { + super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.statisticsService = statisticsService; + this.ruleEngineSettings = ruleEngineSettings; + this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; + this.submitStrategyFactory = submitStrategyFactory; + this.processingStrategyFactory = processingStrategyFactory; + } + + @PostConstruct + public void init() { + super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumerConfigurations.putIfAbsent(configuration.getName(), configuration); + consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); + } + submitExecutor = Executors.newSingleThreadExecutor(); + } + + @PreDestroy + public void stop() { + super.destroy(); + if (submitExecutor != null) { + submitExecutor.shutdownNow(); + } + ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); + } + + @Override + protected void stopMainConsumers() { + consumers.values().forEach(TbQueueConsumer::unsubscribe); + } + + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { + consumersExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); + TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); + + submitStrategy.init(msgs); + + while (!stopped) { + TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); + submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { + log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); + try { + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage())); + } + })); + + boolean timeout = false; + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + timeout = true; + } + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (statsEnabled) { + stats.log(result, decision.isCommit()); + } + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Rule Engine Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { + ToRuleEngineNotificationMsg nfMsg = msg.getValue(); + if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } + + private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes = null; + if (relationTypesList != null) { + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + actorContext.tell(msg, ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + long ts = System.currentTimeMillis(); + consumerStats.forEach((queue, stats) -> { + stats.printStats(); + statisticsService.reportQueueStats(ts, stats); + }); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java new file mode 100644 index 0000000000..aae3cef0ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") +public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { + + private final TenantService tenantService; + + public DefaultTenantRoutingInfoService(TenantService tenantService) { + this.tenantService = tenantService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + Tenant tenant = tenantService.findTenantById(tenantId); + if (tenant != null) { + return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); + } else { + throw new RuntimeException("Tenant not found!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java new file mode 100644 index 0000000000..cc722720d5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.UUID; + +public interface TbClusterService { + + void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); + + void pushNotificationToRuleEngine(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java new file mode 100644 index 0000000000..2dcdb0e0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbCoreConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index dea6e6d33c..c362c5e889 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,31 +21,26 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class RuleEngineStats { +public class TbCoreConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger deviceStateCounter = new AtomicInteger(0); + private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0); + private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0); + public void log(TransportProtos.TransportToDeviceActorMsg msg) { totalCounter.incrementAndGet(); if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -58,9 +53,6 @@ public class RuleEngineStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); - } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } @@ -69,15 +61,32 @@ public class RuleEngineStats { } } + public void log(TransportProtos.DeviceStateServiceMsgProto msg) { + totalCounter.incrementAndGet(); + deviceStateCounter.incrementAndGet(); + } + + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { + totalCounter.incrementAndGet(); + subscriptionMsgCounter.incrementAndGet(); + } + + public void log(TransportProtos.ToCoreNotificationMsg msg) { + totalCounter.incrementAndGet(); + toCoreNotificationsCounter.incrementAndGet(); + } + public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + " deviceState [{}] subMgr [{}] coreNfs [{}]", + total, sessionEventCounter.getAndSet(0), + getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) + , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java new file mode 100644 index 0000000000..f093cc885a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; + +import java.util.UUID; + +@Slf4j +public class TbMsgPackCallback implements TbMsgCallback { + private final UUID id; + private final TenantId tenantId; + private final TbMsgPackProcessingContext ctx; + + public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { + this.id = id; + this.tenantId = tenantId; + this.ctx = ctx; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + ctx.onSuccess(id); + } + + @Override + public void onFailure(RuleEngineException e) { + log.trace("[{}] ON FAILURE", id, e); + ctx.onFailure(tenantId, id, e); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java new file mode 100644 index 0000000000..7e78ac6f5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class TbMsgPackProcessingContext { + + private final TbRuleEngineSubmitStrategy submitStrategy; + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + @Getter + private final ConcurrentMap> pendingMap; + @Getter + private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); + + public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { + this.submitStrategy = submitStrategy; + this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + successMap.put(id, msg); + submitStrategy.onSuccess(id); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + exceptionsMap.putIfAbsent(tenantId, e); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java similarity index 50% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 80674f9bce..5a5c172ee6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -13,35 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.queue; -import lombok.Data; -import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbCallback; -import java.io.Serializable; +import java.util.UUID; -/** - * @author Andrew Shvayka - */ -@Data -@EqualsAndHashCode -public class ServerAddress implements Comparable, Serializable { +@Slf4j +public class TbPackCallback implements TbCallback { + private final TbPackProcessingContext ctx; + private final UUID id; - private final String host; - private final int port; - private final ServerType serverType; + public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + this.id = id; + this.ctx = ctx; + } @Override - public int compareTo(ServerAddress o) { - int result = this.host.compareTo(o.host); - if (result == 0) { - result = this.port - o.port; - } - return result; + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + ctx.onSuccess(id); } @Override - public String toString() { - return '[' + host + ':' + port + ']'; + public void onFailure(Throwable t) { + log.trace("[{}] ON FAILURE", id, t); + ctx.onFailure(id, t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java new file mode 100644 index 0000000000..e9f1224625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbPackProcessingContext { + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + + public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.processingTimeoutLatch = processingTimeoutLatch; + this.pendingCount = new AtomicInteger(ackMap.size()); + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + } + if (empty) { + processingTimeoutLatch.countDown(); + } else { + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T t : ackMap.values()) { + log.trace("left item: {}", t); + } + } + } + } + + public void onFailure(UUID id, Throwable t) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T v : ackMap.values()) { + log.trace("left item: {}", v); + } + } + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public ConcurrentMap getAckMap() { + return ackMap; + } + + public ConcurrentMap getFailedMap() { + return failedMap; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java new file mode 100644 index 0000000000..19966b4566 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbRuleEngineConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java new file mode 100644 index 0000000000..40017d2b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbRuleEngineConsumerStats { + + public static final String TOTAL_MSGS = "totalMsgs"; + public static final String SUCCESSFUL_MSGS = "successfulMsgs"; + public static final String TMP_TIMEOUT = "tmpTimeout"; + public static final String TMP_FAILED = "tmpFailed"; + public static final String TIMEOUT_MSGS = "timeoutMsgs"; + public static final String FAILED_MSGS = "failedMsgs"; + public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; + public static final String FAILED_ITERATIONS = "failedIterations"; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger successIterationsCounter = new AtomicInteger(0); + private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); + + private final String queueName; + + public TbRuleEngineConsumerStats(String queueName) { + this.queueName = queueName; + counters.put(TOTAL_MSGS, totalMsgCounter); + counters.put(SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(FAILED_MSGS, failedMsgCounter); + + counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TMP_FAILED, tmpFailedMsgCounter); + counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); + counters.put(FAILED_ITERATIONS, failedIterationsCounter); + } + + public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { + int success = msg.getSuccessMap().size(); + int pending = msg.getPendingMap().size(); + int failed = msg.getFailedMap().size(); + totalMsgCounter.addAndGet(success + pending + failed); + successMsgCounter.addAndGet(success); + msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); + if (finalIterationForPack) { + if (pending > 0 || failed > 0) { + timeoutMsgCounter.addAndGet(pending); + failedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); + } + failedIterationsCounter.incrementAndGet(); + } else { + successIterationsCounter.incrementAndGet(); + } + } else { + failedIterationsCounter.incrementAndGet(); + tmpTimeoutMsgCounter.addAndGet(pending); + tmpFailedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + } + } + msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); + } + + private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { + ToRuleEngineMsg reMsg = m.getValue(); + return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("] "); + }); + log.info("[{}] Stats: {}", queueName, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + tenantStats.clear(); + tenantExceptions.clear(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java new file mode 100644 index 0000000000..fb36f5064b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbTenantRuleEngineStats { + + private final UUID tenantId; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + + public TbTenantRuleEngineStats(UUID tenantId) { + this.tenantId = tenantId; + counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); + counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); + + counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); + } + + public void logSuccess() { + totalMsgCounter.incrementAndGet(); + successMsgCounter.incrementAndGet(); + } + + public void logFailed() { + totalMsgCounter.incrementAndGet(); + failedMsgCounter.incrementAndGet(); + } + + public void logTimeout() { + totalMsgCounter.incrementAndGet(); + timeoutMsgCounter.incrementAndGet(); + } + + public void logTmpFailed() { + totalMsgCounter.incrementAndGet(); + tmpFailedMsgCounter.incrementAndGet(); + } + + public void logTmpTimeout() { + totalMsgCounter.incrementAndGet(); + tmpTimeoutMsgCounter.incrementAndGet(); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("]"); + }); + log.info("[{}] Stats: {}", tenantId, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java new file mode 100644 index 0000000000..c2705fcbdc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.TbPackCallback; +import org.thingsboard.server.service.queue.TbPackProcessingContext; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractConsumerService implements ApplicationListener { + + protected volatile ExecutorService consumersExecutor; + protected volatile ExecutorService notificationsConsumerExecutor; + protected volatile boolean stopped = false; + + protected final ActorSystemContext actorContext; + protected final DataDecodingEncodingService encodingService; + + protected final TbQueueConsumer> nfConsumer; + + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { + this.actorContext = actorContext; + this.encodingService = encodingService; + this.nfConsumer = nfConsumer; + } + + public void init(String mainConsumerThreadName, String nfConsumerThreadName) { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumers(); + } + + protected abstract ServiceType getServiceType(); + + protected abstract void launchMainConsumers(); + + protected abstract void stopMainConsumers(); + + protected abstract long getNotificationPollDuration(); + + protected abstract long getNotificationPackProcessingTimeout(); + + protected void launchNotificationsConsumer() { + notificationsConsumerExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + handleNotification(id, msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + } + log.info("TB Notifications Consumer stopped."); + }); + } + + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; + + @PreDestroy + public void destroy() { + stopped = true; + + stopMainConsumers(); + + if (nfConsumer != null) { + nfConsumer.unsubscribe(); + } + + if (consumersExecutor != null) { + consumersExecutor.shutdownNow(); + } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..bef733ec22 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { + + protected final String queueName; + protected List orderedMsgList; + private volatile boolean stopped; + + public AbstractTbRuleEngineSubmitStrategy(String queueName) { + this.queueName = queueName; + } + + protected abstract void doOnSuccess(UUID id); + + @Override + public void init(List> msgs) { + orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); + } + + @Override + public ConcurrentMap> getPendingMap() { + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); + for (IdMsgPair pair : orderedMsgList) { + if (reprocessMap.containsKey(pair.uuid)) { + newOrderedMsgList.add(pair); + } + } + orderedMsgList = newOrderedMsgList; + } + + @Override + public void onSuccess(UUID id) { + if (!stopped) { + doOnSuccess(id); + } + } + + @Override + public void stop() { + stopped = true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..d0b1f7f99a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final int batchSize; + private final AtomicInteger packIdx = new AtomicInteger(0); + private final Map> pendingPack = new LinkedHashMap<>(); + private volatile BiConsumer> msgConsumer; + + public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { + super(queueName); + this.batchSize = batchSize; + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + packIdx.set(0); + } + + @Override + protected void doOnSuccess(UUID id) { + boolean endOfPendingPack; + synchronized (pendingPack) { + TbProtoQueueMsg msg = pendingPack.remove(id); + endOfPendingPack = msg != null && pendingPack.isEmpty(); + } + if (endOfPendingPack) { + packIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int startIdx = Math.min(packIdx.get() * batchSize, listSize); + int endIdx = Math.min(startIdx + batchSize, listSize); + synchronized (pendingPack) { + pendingPack.clear(); + for (int i = startIdx; i < endIdx; i++) { + IdMsgPair pair = orderedMsgList.get(i); + pendingPack.put(pair.uuid, pair.msg); + } + } + int submitSize = pendingPack.size(); + if (log.isInfoEnabled() && submitSize > 0) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + } + pendingPack.forEach(msgConsumer); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ffd1dd49d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + public BurstTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + } + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + } + + @Override + protected void doOnSuccess(UUID id) { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java similarity index 64% rename from application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java index 9f08463f91..2b2c203ec5 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue.processing; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.io.IOException; +import java.util.UUID; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { - @Override - public ToRuleEngineMsg decode(byte[] data) throws IOException { - return ToRuleEngineMsg.parseFrom(data); +public class IdMsgPair { + final UUID uuid; + final TbProtoQueueMsg msg; + + public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { + this.uuid = uuid; + this.msg = msg; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ae5993cb1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,108 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private volatile BiConsumer> msgConsumer; + private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); + + public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void init(List> msgs) { + super.init(msgs); + initMaps(); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + entityIdToListMap.forEach((entityId, queue) -> { + IdMsgPair msg = queue.peek(); + if (msg != null) { + msgConsumer.accept(msg.uuid, msg.msg); + } + }); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + initMaps(); + } + + @Override + protected void doOnSuccess(UUID id) { + EntityId entityId = msgToEntityIdMap.get(id); + if (entityId != null) { + Queue queue = entityIdToListMap.get(entityId); + if (queue != null) { + IdMsgPair next = null; + synchronized (queue) { + IdMsgPair expected = queue.peek(); + if (expected != null && expected.uuid.equals(id)) { + queue.poll(); + next = queue.peek(); + } + } + if (next != null) { + msgConsumer.accept(next.uuid, next.msg); + } + } + } + } + + private void initMaps() { + msgToEntityIdMap.clear(); + entityIdToListMap.clear(); + for (IdMsgPair pair : orderedMsgList) { + EntityId entityId = getEntityId(pair.msg.getValue()); + if (entityId != null) { + msgToEntityIdMap.put(pair.uuid, entityId); + entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); + } + } + } + + protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..cd8a97e82c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@Slf4j +public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + try { + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } catch (InvalidProtocolBufferException e) { + log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); + return null; + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java similarity index 55% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index 3326c33aa3..b258c6db1b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -13,28 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.queue.processing; -import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; -@Data -public class SendToClusterMsg implements TbActorMsg { +import java.util.UUID; - private TbActorMsg msg; - private EntityId entityId; +public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { - public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { - this.entityId = entityId; - this.msg = msg; + public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); } - @Override - public MsgType getMsgType() { - return MsgType.SEND_TO_CLUSTER_MSG; + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ef45b983fc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final AtomicInteger msgIdx = new AtomicInteger(0); + private volatile BiConsumer> msgConsumer; + private volatile UUID expectedMsgId; + + public SequentialTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + msgIdx.set(0); + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + } + + @Override + protected void doOnSuccess(UUID id) { + if (expectedMsgId.equals(id)) { + msgIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int idx = msgIdx.get(); + if (idx < listSize) { + IdMsgPair pair = orderedMsgList.get(idx); + expectedMsgId = pair.uuid; + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + } + msgConsumer.accept(pair.uuid, pair.msg); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java new file mode 100644 index 0000000000..4a4829f0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Data; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +@Data +public class TbRuleEngineProcessingDecision { + + private final boolean commit; + private final ConcurrentMap> reprocessMap; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java new file mode 100644 index 0000000000..e818ffb9d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +public class TbRuleEngineProcessingResult { + + @Getter + private final boolean success; + @Getter + private final boolean timeout; + @Getter + private final TbMsgPackProcessingContext ctx; + + public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { + this.timeout = timeout; + this.ctx = ctx; + this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); + } + + public ConcurrentMap> getPendingMap() { + return ctx.getPendingMap(); + } + + public ConcurrentMap> getSuccessMap() { + return ctx.getSuccessMap(); + } + + public ConcurrentMap> getFailedMap() { + return ctx.getFailedMap(); + } + + public ConcurrentMap getExceptionsMap() { + return ctx.getExceptionsMap(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java new file mode 100644 index 0000000000..cfba85f5dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +public interface TbRuleEngineProcessingStrategy { + + TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java new file mode 100644 index 0000000000..bbf283e962 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -0,0 +1,140 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class TbRuleEngineProcessingStrategyFactory { + + public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "SKIP_ALL_FAILURES": + return new SkipStrategy(name); + case "RETRY_ALL": + return new RetryStrategy(name, true, true, true, configuration); + case "RETRY_FAILED": + return new RetryStrategy(name, false, true, false, configuration); + case "RETRY_TIMED_OUT": + return new RetryStrategy(name, false, false, true, configuration); + case "RETRY_FAILED_AND_TIMED_OUT": + return new RetryStrategy(name, false, true, true, configuration); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + + private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; + private final boolean retrySuccessful; + private final boolean retryFailed; + private final boolean retryTimeout; + private final int maxRetries; + private final double maxAllowedFailurePercentage; + private final long pauseBetweenRetries; + + private int initialTotalCount; + private int retryCount; + + public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + this.queueName = queueName; + this.retrySuccessful = retrySuccessful; + this.retryFailed = retryFailed; + this.retryTimeout = retryTimeout; + this.maxRetries = configuration.getRetries(); + this.maxAllowedFailurePercentage = configuration.getFailurePercentage(); + this.pauseBetweenRetries = configuration.getPauseBetweenRetries(); + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (result.isSuccess()) { + return new TbRuleEngineProcessingDecision(true, null); + } else { + if (retryCount == 0) { + initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); + } + retryCount++; + double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); + if (maxRetries > 0 && retryCount > maxRetries) { + log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { + log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else { + ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); + if (retryFailed) { + result.getFailedMap().forEach(toReprocess::put); + } + if (retryTimeout) { + result.getPendingMap().forEach(toReprocess::put); + } + if (retrySuccessful) { + result.getSuccessMap().forEach(toReprocess::put); + } + log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + if (log.isTraceEnabled()) { + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (pauseBetweenRetries > 0) { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return new TbRuleEngineProcessingDecision(false, toReprocess); + } + } + } + } + + private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + + private final String queueName; + + public SkipStrategy(String name) { + this.queueName = name; + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (!result.isSuccess()) { + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + return new TbRuleEngineProcessingDecision(true, null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..7b22da97db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +public interface TbRuleEngineSubmitStrategy { + + void init(List> msgs); + + ConcurrentMap> getPendingMap(); + + void submitAttempt(BiConsumer> msgConsumer); + + void update(ConcurrentMap> reprocessMap); + + void onSuccess(UUID id); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java new file mode 100644 index 0000000000..90eb8561a0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; + +@Component +@Slf4j +public class TbRuleEngineSubmitStrategyFactory { + + public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "BURST": + return new BurstTbRuleEngineSubmitStrategy(name); + case "BATCH": + return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); + case "SEQUENTIAL_BY_ORIGINATOR": + return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL_BY_TENANT": + return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL": + return new SequentialTbRuleEngineSubmitStrategy(name); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java deleted file mode 100644 index 5d303b54ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -public class DefaultDeviceRpcService implements DeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DeviceService deviceService; - - @Autowired - @Lazy - private ActorService actorService; - - private ScheduledExecutorService rpcCallBackExecutor; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - @PostConstruct - public void initExecutor() { - rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (rpcCallBackExecutor != null) { - rpcCallBackExecutor.shutdownNow(); - } - } - - @Override - public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } else { - ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); - builder.setRequestIdMSB(response.getId().getMostSignificantBits()); - builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); - response.getResponse().ifPresent(builder::setResponse); - if (response.getError().isPresent()) { - builder.setError(response.getError().get().ordinal()); - } else { - builder.setError(-1); - } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); - } - } - - @Override - public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId, localToDeviceRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = localToDeviceRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.FromDeviceRPCResponseProto proto; - try { - proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); - } - - @Override - public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { - ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); - forward(deviceId, rpcMsg); - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); - } - - private void forward(DeviceId deviceId, T msg) { - actorService.onMsg(new SendToClusterMsg(deviceId, msg)); - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - rpcCallBackExecutor.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = requestsMap.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java new file mode 100644 index 0000000000..f6e5eb8b43 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import akka.actor.ActorRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +@TbCoreComponent +public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + private final DeviceService deviceService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + private final ActorSystemContext actorContext; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbRuleEngineRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, + ActorSystemContext actorContext) { + this.deviceService = deviceService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + this.actorContext = actorContext; + } + + @Autowired(required = false) + public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { + this.tbRuleEngineRpcService = tbRuleEngineRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleToRuleEngineTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { + ToDeviceRpcRequest request = rpcMsg.getMsg(); + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, rpcMsg); + actorContext.tell(rpcMsg, ActorRef.noSender()); + scheduleToDeviceTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); + if (request != null) { + sendRpcResponseToTbRuleEngine(request.getServiceId(), response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbRuleEngineRpcService.isPresent()) { + tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.pushNotificationToRuleEngine(originServiceId, response, null); + } + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originServiceId", serviceId); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to rule engine request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for processing to rule engine request.", requestId); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to device request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to device request.", requestId); + localToDeviceRpcRequests.remove(requestId); + }, timeout, TimeUnit.MILLISECONDS); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java new file mode 100644 index 0000000000..0ec730b7dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { + + private final PartitionService partitionService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + + + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbCoreRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbRuleEngineRpcService(PartitionService partitionService, + TbClusterService clusterService, + TbServiceInfoProvider serviceInfoProvider) { + this.partitionService = partitionService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Autowired(required = false) + public void setTbCoreRpcService(Optional tbCoreRpcService) { + this.tbCoreRpcService = tbCoreRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { + TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId) + .setPayload(body).build(); + TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setToServerResponse(responseMsg) + .build(); + clusterService.pushNotificationToTransport(serviceId, msg, null); + } + + @Override + public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + forwardRpcRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + sendRpcResponseToTbCore(src.getOriginServiceId(), response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + + @Override + public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(response)); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + toDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId); + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); + if (tpi.isMyPartition()) { + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); + clusterService.pushMsgToCore(rpcMsg, null); + } + } + + private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.pushNotificationToCore(originServiceId, response, null); + } + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); + } + }, timeout, TimeUnit.MILLISECONDS); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java deleted file mode 100644 index feb5d58c3d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 16.04.18. - */ -public interface DeviceRpcService { - - void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); - - void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); - - void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); - - void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index 78ea0c5660..c1e5e2e038 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,7 +19,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java new file mode 100644 index 0000000000..896745c48e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Handles REST API calls that contain RPC requests to Device. + */ +public interface TbCoreDeviceRpcService { + + /** + * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. + * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} + * + * @param request the RPC request + * @param responseConsumer the consumer of the RPC response + */ + void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); + + /** + * Handles the RPC response from the Rule Engine. + * + * @param response the RPC response + */ + void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); + + /** + * Forwards the RPC request from Rule Engine to Device Actor + * + * @param request the RPC request message + */ + void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java new file mode 100644 index 0000000000..3d4f23a796 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.rule.engine.api.RuleEngineRpcService; + +/** + * Created by ashvayka on 16.04.18. + */ +public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDevice(FromDeviceRpcResponse response); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index ba537aed25..3cfe9bc1f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,11 +22,8 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import java.util.Optional; - /** * Created by ashvayka on 16.04.18. */ @@ -35,7 +32,7 @@ import java.util.Optional; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final ServerAddress serverAddress; + private final String serviceId; @Getter private final ToDeviceRpcRequest msg; diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 31827f542e..daa74d013f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,25 +78,53 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); + protected abstract long getMaxBlacklistDuration(); + protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - switch (scriptType) { - case RULE_NODE_SCRIPT: - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); - default: - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); + if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); } + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - if (blackListedFunctions.containsKey(scriptId)) { - AtomicInteger errorCount = blackListedFunctions.get(scriptId); - return errorCount.get() >= getMaxErrors(); + BlackListInfo errorCount = blackListedFunctions.get(scriptId); + if (errorCount != null) { + if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { + blackListedFunctions.remove(scriptId); + return false; + } else { + return errorCount.get() >= getMaxErrors(); + } } else { return false; } } + + private class BlackListInfo { + private final AtomicInteger counter; + private long expirationTime; + + private BlackListInfo() { + this.counter = new AtomicInteger(0); + } + + public int get() { + return counter.get(); + } + + public int incrementAndGet() { + int result = counter.incrementAndGet(); + expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); + return result; + } + + public long getExpirationTime() { + return expirationTime; + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 49a6304ebb..a9acfa4f42 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 909565609c..66a14cc827 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; + @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -37,6 +39,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; + @Value("${js.local.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -56,4 +61,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 1a218b119b..8d1f9d662a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -23,15 +23,13 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -39,42 +37,25 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) +@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Value("${js.remote.request_topic}") - private String requestTopic; - - @Value("${js.remote.response_topic_prefix}") - private String responseTopicPrefix; - - @Value("${js.remote.max_pending_requests}") - private long maxPendingRequests; - @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; - @Value("${js.remote.response_poll_interval}") - private int responsePollDuration; - - @Value("${js.remote.response_auto_commit_interval}") - private int autoCommitInterval; - @Getter @Value("${js.remote.max_errors}") private int maxErrors; + @Value("${js.remote.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -99,42 +80,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private TbKafkaRequestTemplate kafkaTemplate; + @Autowired + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(requestTopic); - requestBuilder.encoder(new RemoteJsRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new RemoteJsResponseDecoder()); - responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - kafkaTemplate = builder.build(); - kafkaTemplate.init(); + requestTemplate.init(); } @PreDestroy public void destroy() { - if (kafkaTemplate != null) { - kafkaTemplate.stop(); + if (requestTemplate != null) { + requestTemplate.stop(); } } @@ -151,11 +110,12 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaEvalMsgs.incrementAndGet(); } @@ -168,7 +128,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -202,16 +162,17 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaInvokeMsgs.incrementAndGet(); } @Override public void onFailure(Throwable t) { + onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -219,10 +180,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { + onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -240,8 +202,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); - JsInvokeProtos.RemoteJsResponse response = future.get(); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); @@ -252,4 +214,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index d07a2490ba..51ecba27b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,19 +17,20 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder { +public class RemoteJsRequestEncoder implements TbKafkaEncoder> { @Override - public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { + public byte[] encode(TbProtoQueueMsg value) { try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 7a3876fb91..621d2b05d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,8 +16,10 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -25,12 +27,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder { +public class RemoteJsResponseDecoder implements TbKafkaDecoder> { @Override - public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { + public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); - return builder.build(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index ef5d4716cb..110f410fad 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index e359befb30..572c3a6ed0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -46,6 +46,7 @@ import ua_parser.Client; import java.util.UUID; + @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c58a664790..c13cb5529a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.security.device; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsFilter; @@ -24,16 +23,21 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; @Service +@TbCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - @Autowired - DeviceService deviceService; + private final DeviceService deviceService; - @Autowired - DeviceCredentialsService deviceCredentialsService; + private final DeviceCredentialsService deviceCredentialsService; + + public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { + this.deviceService = deviceService; + this.deviceCredentialsService = deviceCredentialsService; + } @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 40d19da764..5412ed6def 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,11 +15,9 @@ */ package org.thingsboard.server.service.security.permission; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index 9777af7b8e..a8c5dc13ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -16,20 +16,13 @@ package org.thingsboard.server.service.security.permission; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 952c63c9b9..1b9d566206 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,9 +21,9 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Collections; -import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @@ -31,6 +31,7 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service +@TbCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 99a24cfb12..24eea18b53 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.state; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; @@ -23,16 +22,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -49,16 +45,18 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -69,7 +67,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -89,8 +86,8 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service +@TbCoreComponent @Slf4j -//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -104,31 +101,15 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - @Autowired - private TenantService tenantService; - - @Autowired - private DeviceService deviceService; + private final TenantService tenantService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService tsService; + private final TbClusterService clusterService; + private final PartitionService partitionService; - @Autowired - private AttributesService attributesService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired private TelemetrySubscriptionService tsSubService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -148,18 +129,32 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); - private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + + public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, + AttributesService attributesService, TimeseriesService tsService, + TbClusterService clusterService, PartitionService partitionService) { + this.tenantService = tenantService; + this.deviceService = deviceService; + this.attributesService = attributesService; + this.tsService = tsService; + this.clusterService = clusterService; + this.partitionService = partitionService; + } + + @Autowired + public void setTsSubService(TelemetrySubscriptionService tsSubService) { + this.tsSubService = tsSubService; + } @PostConstruct public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); - queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); - //TODO: schedule persistence in v2.1; } @PreDestroy @@ -171,133 +166,210 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - queueExecutor.submit(() -> onDeviceAddedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override public void onDeviceUpdated(Device device) { - queueExecutor.submit(() -> onDeviceUpdatedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); } @Override - public void onDeviceConnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); + public void onDeviceDeleted(Device device) { + sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); } @Override - public void onDeviceActivity(DeviceId deviceId) { - deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); + public void onDeviceConnect(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } } @Override - public void onDeviceDisconnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); + public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { + deviceLastReportedActivity.put(deviceId, lastReportedActivity); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } } @Override - public void onDeviceDeleted(Device device) { - queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); + public void onDeviceDisconnect(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); - } - - @Override - public void onClusterUpdate() { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(this::onClusterUpdateSync); + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } } } @Override - public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.DeviceStateServiceMsgProto proto; + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { try { - proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - onDeviceAdded(device); - } else if (proto.getUpdated()) { - onDeviceUpdated(device); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + onDeviceDeleted(tenantId, deviceId); + callback.onSuccess(); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); + if (partitionedDevices.containsKey(tpi)) { + addDeviceUsingState(tpi, state); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , tenantId, deviceId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + callback.onFailure(t); + } + }, MoreExecutors.directExecutor()); + } else if (proto.getUpdated()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } + } else { + //Device was probably deleted while message was in queue; + callback.onSuccess(); } } + } catch (Exception e) { + log.trace("Failed to process queue msg: [{}]", proto, e); + callback.onFailure(e); } } - private void onClusterUpdateSync() { - clusterUpdatePending = false; - List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); - for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); - PageLink pageLink = new PageLink(initFetchPackSize); - while (pageLink != null) { - PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); - pageLink = page.hasNext() ? pageLink.nextPageLink() : null; - for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } - } else { - Set tenantDeviceSet = tenantDevices.get(tenant.getId()); - if (tenantDeviceSet != null) { - tenantDeviceSet.remove(device.getId()); - } - deviceStates.remove(device.getId()); - deviceLastReportedActivity.remove(device.getId()); - deviceLastSavedActivity.remove(device.getId()); - } - } - try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); - } catch (InterruptedException | ExecutionException e) { - log.warn("Failed to init device state service from DB", e); + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + synchronized (this) { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(() -> { + clusterUpdatePending = false; + initStateFromDB(partitionChangeEvent.getPartitions()); + }); } } } } - private void initStateFromDB() { + private void initStateFromDB(Set partitions) { try { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(partitionedDevices.keySet()); + + Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); + removedPartitions.removeAll(partitions); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set devices = partitionedDevices.remove(partition); + devices.forEach(deviceId -> { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + }); + }); + + addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); + + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. + // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); PageLink pageLink = new PageLink(initFetchPackSize); while (pageLink != null) { + List> fetchFutures = new ArrayList<>(); PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.hasNext() ? pageLink.nextPageLink() : null; for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); + if (addedPartitions.contains(tpi)) { + ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { + @Nullable + @Override + public Void apply(@Nullable DeviceStateData state) { + if (state != null) { + addDeviceUsingState(tpi, state); + } + return null; + } + }, MoreExecutors.directExecutor()); + fetchFutures.add(future); } } try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + Futures.successfulAsList(fetchFutures).get(); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } } } + log.info("Managing following partitions:"); + partitionedDevices.forEach((tpi, devices) -> { + log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); + }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } } - private void addDeviceUsingState(DeviceStateData state) { - tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { + partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } @@ -325,103 +397,24 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void onDeviceConnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } - } - - private void onDeviceDisconnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } - } - - private void onDeviceActivitySync(DeviceId deviceId) { - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - } - private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - if (!routingService.resolveById(deviceId).isPresent()) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); - } + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); } } } return deviceStateData; } - private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - - private void onDeviceAddedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - } - }, MoreExecutors.directExecutor()); - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); - } - } - - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { - log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); - ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { + TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -429,40 +422,17 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); - } - - private void onDeviceUpdatedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); - } + TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); + clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - Optional address = routingService.resolveById(deviceId); - if (!address.isPresent()) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } - } - } else { - sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); - } + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + Set deviceIdSet = partitionedDevices.get(tpi); + deviceIdSet.remove(deviceId); } private ListenableFuture fetchDeviceState(Device device) { @@ -523,10 +493,9 @@ public class DefaultDeviceStateService implements DeviceStateService { private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) { DeviceState state = stateData.getState(); try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON - , json.writeValueAsString(state) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON + , json.writeValueAsString(state)); + clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } @@ -554,7 +523,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private class AttributeSaveCallback implements FutureCallback { + private static class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 8dbe7579e3..1665a27b84 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.service.state; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.TbCallback; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService { +public interface DeviceStateService extends ApplicationListener { void onDeviceAdded(Device device); @@ -32,13 +35,12 @@ public interface DeviceStateService { void onDeviceConnect(DeviceId deviceId); - void onDeviceActivity(DeviceId deviceId); + void onDeviceActivity(DeviceId deviceId, long lastReportedActivityTime); void onDeviceDisconnect(DeviceId deviceId); void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout); - void onClusterUpdate(); + void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback bytes); - void onRemoteMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java new file mode 100644 index 0000000000..a506b87ab0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -0,0 +1,141 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.stats; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { + + public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; + public static final FutureCallback CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to persist statistics", t); + } + }; + + private final TbServiceInfoProvider serviceInfoProvider; + private final TelemetrySubscriptionService tsService; + private final Lock lock = new ReentrantLock(); + private final AssetService assetService; + private final ConcurrentMap tenantQueueAssets; + + public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { + this.tsService = tsService; + this.serviceInfoProvider = serviceInfoProvider; + this.assetService = assetService; + this.tenantQueueAssets = new ConcurrentHashMap<>(); + } + + @Override + public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { + String queueName = ruleEngineStats.getQueueName(); + ruleEngineStats.getTenantStats().forEach((id, stats) -> { + TenantId tenantId = new TenantId(id); + try { + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + } catch (DataValidationException e) { + if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e; + } + } + }); + ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { + TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); + try { + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + } catch (DataValidationException e2) { + if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e2; + } + } + }); + ruleEngineStats.reset(); + } + + private AssetId getServiceAssetId(TenantId tenantId, String queueName) { + TenantQueueKey key = new TenantQueueKey(tenantId, queueName); + AssetId assetId = tenantQueueAssets.get(key); + if (assetId == null) { + lock.lock(); + try { + assetId = tenantQueueAssets.get(key); + if (assetId == null) { + Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); + if (asset == null) { + asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); + asset.setType(TB_SERVICE_QUEUE); + asset = assetService.saveAsset(asset); + } + assetId = asset.getId(); + tenantQueueAssets.put(key, assetId); + } + } finally { + lock.unlock(); + } + } + return assetId; + } + + @Data + private static class TenantQueueKey { + private final TenantId tenantId; + private final String queueName; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java new file mode 100644 index 0000000000..88573844c9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.stats; + +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; + +public interface RuleEngineStatisticsService { + + void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java new file mode 100644 index 0000000000..67c81ac06f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -0,0 +1,380 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.function.Predicate; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultSubscriptionManagerService implements SubscriptionManagerService { + + @Autowired + private AttributesService attrService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + + @Autowired + private TbQueueProducerProvider producerProvider; + + @Autowired + private TbLocalSubscriptionService localSubscriptionService; + + @Autowired + private DeviceStateService deviceStateService; + + @Autowired + private TbClusterService clusterService; + + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + + private ExecutorService tsCallBackExecutor; + private String serviceId; + private TbQueueProducer> toCoreNotificationsProducer; + + @PostConstruct + public void initExecutor() { + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + serviceId = serviceInfoProvider.getServiceId(); + toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (tsCallBackExecutor != null) { + tsCallBackExecutor.shutdownNow(); + } + } + + @Override + public void addSubscription(TbSubscription subscription, TbCallback callback) { + log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", + subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); + } + boolean newSubscription = subscriptionsByEntityId + .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); + subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); + if (newSubscription) { + switch (subscription.getType()) { + case TIMESERIES: + handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); + break; + case ATTRIBUTES: + handleNewAttributeSubscription((TbAttributeSubscription) subscription); + break; + } + } + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + removeSubscriptionFromEntityMap(subscription); + removeSubscriptionFromPartitionMap(subscription); + if (sessionSubscriptions.isEmpty()) { + subscriptionsByWsSessionId.remove(sessionId); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } + } + + @Override + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { + return (TbTimeseriesSubscription) s; + } else { + return null; + } + }, s -> true, s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + callback.onSuccess(); + } + + @Override + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { + return (TbAttributeSubscription) s; + } else { + return null; + } + }, + s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), + s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + if (entityId.getEntityType() == EntityType.DEVICE) { + if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } + } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { + clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) + , null); + } + } + callback.onSuccess(); + } + + private void onLocalSubUpdate(EntityId entityId, + Function castFunction, + Predicate filterFunction, + Function> processFunction) { + Set entitySubscriptions = subscriptionsByEntityId.get(entityId); + if (entitySubscriptions != null) { + entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { + List subscriptionUpdate = processFunction.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + if (serviceId.equals(s.getServiceId())) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); + } else { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void removeSubscriptionFromEntityMap(TbSubscription sub) { + Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); + if (entitySubSet != null) { + entitySubSet.remove(sub); + if (entitySubSet.isEmpty()) { + subscriptionsByEntityId.remove(sub.getEntityId()); + } + } + } + + private void removeSubscriptionFromPartitionMap(TbSubscription sub) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); + Set subs = partitionedSubscriptions.get(tpi); + if (subs != null) { + subs.remove(sub); + } + } + + private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } + + private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().forEach((key, value) -> { + if (curTs > value) { + long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); + long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; + queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); + + builder.setSessionId(subscription.getSessionId()); + builder.setSubscriptionId(subscription.getSubscriptionId()); + + Map> data = new TreeMap<>(); + for (TsKvEntry tsEntry : updates) { + List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); + Object[] value = new Object[2]; + value[0] = tsEntry.getTs(); + value[1] = tsEntry.getValueAsString(); + values.add(value); + } + + data.forEach((key, value) -> { + TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); + dataBuilder.setKey(key); + value.forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + builder.addData(dataBuilder.build()); + }); + + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( + LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) + .build(); + return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java new file mode 100644 index 0000000000..d57067fa28 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { + + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); + + @Autowired + private TelemetryWebSocketService wsService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbClusterService clusterService; + + @Autowired + @Lazy + private SubscriptionManagerService subscriptionManagerService; + + private ExecutorService wsCallBackExecutor; + + @PostConstruct + public void initExecutor() { + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + } + + @PreDestroy + public void shutdownExecutor() { + if (wsCallBackExecutor != null) { + wsCallBackExecutor.shutdownNow(); + } + } + + @Override + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + } + } + + @Override + @EventListener(ClusterTopologyChangeEvent.class) + public void onApplicationEvent(ClusterTopologyChangeEvent event) { + if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { + /* + * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. + * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. + * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically + * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. + * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache + * Since number of subscriptions is usually much less then number of devices that are pushing data. + */ + subscriptionsBySessionId.values().forEach(map -> map.values() + .forEach(sub -> pushSubscriptionToManagerService(sub, false))); + } + } + + //TODO 3.1: replace null callbacks with callbacks from websocket service. + @Override + public void addSubscription(TbSubscription subscription) { + EntityId entityId = subscription.getEntityId(); + // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { + subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); + } + pushSubscriptionToManagerService(subscription, true); + registerSubscription(subscription); + } + + private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + if (pushToLocalService) { + subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); + } + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); + } + } + + @Override + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { + TbSubscription subscription = subscriptionsBySessionId + .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); + if (subscription != null) { + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; + update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); + break; + case ATTRIBUTES: + TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; + update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); + break; + } + wsService.sendWsMsg(sessionId, update); + } + callback.onSuccess(); + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + if (sessionSubscriptions.isEmpty()) { + subscriptionsBySessionId.remove(sessionId); + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + } + + @Override + public void cancelAllSessionSubscriptions(String sessionId) { + Map subscriptions = subscriptionsBySessionId.get(sessionId); + if (subscriptions != null) { + Set toRemove = new HashSet<>(subscriptions.keySet()); + toRemove.forEach(id -> cancelSubscription(sessionId, id)); + } + } + + private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); + + Map keyStates; + if (subscription.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = subscription.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + return TbTimeseriesSubscription.builder() + .serviceId(subscription.getServiceId()) + .sessionId(subscription.getSessionId()) + .subscriptionId(subscription.getSubscriptionId()) + .tenantId(subscription.getTenantId()) + .entityId(entityView.getEntityId()) + .startTime(entityView.getStartTimeMs()) + .endTime(entityView.getEndTimeMs()) + .allKeys(false) + .keyStates(keyStates).build(); + } + + private void registerSubscription(TbSubscription subscription) { + Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java new file mode 100644 index 0000000000..e20b707348 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.List; + +public interface SubscriptionManagerService extends ApplicationListener { + + void addSubscription(TbSubscription subscription, TbCallback callback); + + void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); + + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); + + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java new file mode 100644 index 0000000000..83a86efefc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbAttributeSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final TbAttributeSubscriptionScope scope; + + @Builder + public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.scope = scope; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java index 095e10f0b3..d23cc3581d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.service.subscription; -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { +public enum TbAttributeSubscriptionScope { + + CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE - ServerInstance getSelf(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java new file mode 100644 index 0000000000..58fba24250 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +public interface TbLocalSubscriptionService { + + void addSubscription(TbSubscription subscription); + + void cancelSubscription(String sessionId, int subscriptionId); + + void cancelAllSessionSubscriptions(String sessionId); + + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); + + void onApplicationEvent(PartitionChangeEvent event); + + void onApplicationEvent(ClusterTopologyChangeEvent event); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java new file mode 100644 index 0000000000..22b37ff690 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@Data +@AllArgsConstructor +public abstract class TbSubscription { + + private final String serviceId; + private final String sessionId; + private final int subscriptionId; + private final TenantId tenantId; + private final EntityId entityId; + private final TbSubscriptionType type; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbSubscription that = (TbSubscription) o; + return subscriptionId == that.subscriptionId && + sessionId.equals(that.sessionId) && + tenantId.equals(that.tenantId) && + entityId.equals(that.entityId) && + type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java similarity index 82% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java index 4b00284c34..d7a4715460 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.subscription; -/** - * Created by ashvayka on 23.09.18. - */ -public enum ServerType { - CORE +public enum TbSubscriptionType { + TIMESERIES, ATTRIBUTES } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java new file mode 100644 index 0000000000..d0b4d713f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java @@ -0,0 +1,247 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +public class TbSubscriptionUtils { + + public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() + .setServiceId(subscription.getServiceId()) + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()) + .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(subscription.getEntityId().getEntityType().name()) + .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); + + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; + TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(tSub.isAllKeys()); + tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + tSubProto.setStartTime(tSub.getStartTime()); + tSubProto.setEndTime(tSub.getEndTime()); + msgBuilder.setTelemetrySub(tSubProto.build()); + break; + case ATTRIBUTES: + TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; + TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(aSub.isAllKeys()) + .setScope(aSub.getScope().name()); + aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + msgBuilder.setAttributeSub(aSubProto.build()); + break; + } + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()).build(); + msgBuilder.setSubClose(closeProto); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { + TbSubscriptionProto subProto = attributeSub.getSub(); + TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); + builder.allKeys(attributeSub.getAllKeys()); + Map keyStates = new HashMap<>(); + attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.keyStates(keyStates); + return builder.build(); + } + + public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { + TbSubscriptionProto subProto = telemetrySub.getSub(); + TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.allKeys(telemetrySub.getAllKeys()); + Map keyStates = new HashMap<>(); + telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.startTime(telemetrySub.getStartTime()); + builder.endTime(telemetrySub.getEndTime()); + builder.keyStates(keyStates); + return builder.build(); + } + + public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { + TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setTsUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setAttrUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { + KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); + switch (attr.getDataType()) { + case BOOLEAN: + attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); + break; + case LONG: + attr.getLongValue().ifPresent(dataBuilder::setLongV); + break; + case DOUBLE: + attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); + break; + case JSON: + attr.getJsonValue().ifPresent(dataBuilder::setJsonV); + break; + case STRING: + attr.getStrValue().ifPresent(dataBuilder::setStringV); + break; + } + return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); + } + + public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { + return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); + } + + public static List toTsKvEntityList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); + return result; + } + + public static List toAttributeKvList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); + return result; + } + + private static KvEntry getKvEntry(KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getType().getNumber()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongV()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStringV()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); + break; + } + return entry; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java new file mode 100644 index 0000000000..0be63f7b65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbTimeseriesSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final long startTime; + @Getter private final long endTime; + + @Builder + public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, long startTime, long endTime) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d29e5de5b7..665a787e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,73 +18,43 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.Subscription; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -93,39 +63,36 @@ import java.util.stream.Collectors; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - @Autowired - private TelemetryWebSocketService wsService; + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - @Autowired - private AttributesService attrService; + private final AttributesService attrService; + private final TimeseriesService tsService; + private final TbClusterService clusterService; + private final PartitionService partitionService; + private Optional subscriptionManagerService; - @Autowired - private TimeseriesService tsService; - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private EntityViewService entityViewService; - - @Autowired - @Lazy - private DeviceStateService stateService; - - @Autowired - @Lazy - private ActorService actorService; - private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; + public DefaultTelemetrySubscriptionService(AttributesService attrService, + TimeseriesService tsService, + TbClusterService clusterService, + PartitionService partitionService) { + this.attrService = attrService; + this.tsService = tsService; + this.clusterService = clusterService; + this.partitionService = partitionService; + } + + @Autowired(required = false) + public void setSubscriptionManagerService(Optional subscriptionManagerService) { + this.subscriptionManagerService = subscriptionManagerService; + } + @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); } @PreDestroy @@ -138,64 +105,12 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - @Override - public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { - long startTime = 0L; - long endTime = 0L; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); - entityId = entityView.getEntityId(); - startTime = entityView.getStartTimeMs(); - endTime = entityView.getEndTimeMs(); - sub = getUpdatedSubscriptionState(entityId, sub, entityView); - } - Optional server = routingService.resolveById(entityId); - Subscription subscription; - if (server.isPresent()) { - ServerAddress address = server.get(); - log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); - subscription = new Subscription(sub, true, address, startTime, endTime); - tellNewSubscription(address, sessionId, subscription); - } else { - log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); - subscription = new Subscription(sub, true, null, startTime, endTime); - } - registerSubscription(sessionId, entityId, subscription); - } - - private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { - Map keyStates; - if (sub.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = sub.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); - } - - @Override - public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { - cleanupLocalWsSessionSubscriptions(sessionId); - } - - @Override - public void removeSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - Subscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); } } @@ -208,14 +123,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); + addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); } @Override @@ -242,352 +157,31 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - @Override - public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } - - @Override - public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionProto proto; - try { - proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - Map statesMap = proto.getKeyStatesList().stream().collect( - Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); - Subscription subscription = new Subscription( - new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), - new TenantId(UUID.fromString(proto.getTenantId())), - EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), - false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); - - addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); - } - - @Override - public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionUpdateProto proto; - try { - proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - SubscriptionUpdate update = convert(proto); - String sessionId = proto.getSessionId(); - log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); - Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); - if (subOpt.isPresent()) { - updateSubscriptionState(sessionId, subOpt.get(), update); - wsService.sendWsMsg(sessionId, update); - } - } - - @Override - public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionCloseProto proto; - try { - proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); - } - - @Override - public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SessionCloseProto proto; - try { - proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); - } - - @Override - public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.AttributeUpdateProto proto; - try { - proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), - proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); - } - - @Override - public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.TimeseriesUpdateProto proto; - try { - proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); - } - - @Override - public void onClusterUpdate() { - log.trace("Processing cluster onUpdate msg!"); - Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); - while (deviceIterator.hasNext()) { - Map.Entry> e = deviceIterator.next(); - Set subscriptions = e.getValue(); - Optional newAddressOptional = routingService.resolveById(e.getKey()); - if (newAddressOptional.isPresent()) { - newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); } else { - checkSubscriptionsPrevAddress(subscriptions); - } - if (subscriptions.size() == 0) { - log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); - deviceIterator.remove(); - } - } - } - - private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { - Iterator subscriptionIterator = subscriptions.iterator(); - while (subscriptionIterator.hasNext()) { - Subscription s = subscriptionIterator.next(); - if (s.isLocal()) { - if (!newAddress.equals(s.getServer())) { - log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); - s.setServer(newAddress); - tellNewSubscription(newAddress, s.getWsSessionId(), s); - } - } else { - log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); - subscriptionIterator.remove(); - //TODO: onUpdate state of subscription by WsSessionId and other maps. - } - } - } - - private void checkSubscriptionsPrevAddress(Set subscriptions) { - for (Subscription s : subscriptions) { - if (s.isLocal() && s.getServer() != null) { - log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); - s.setServer(null); - } else { - log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); - } - } - } - - private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); - registerSubscription(sessionId, entityId, subscription); - if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().entrySet().forEach(e -> { - if (curTs > e.getValue()) { - queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); - } else { - log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - } - - private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - Optional serverAddress = routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalAttributesUpdate(entityId, scope, attributes); - if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } } else { - tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); - } - } - - private void onTimeseriesUpdate(EntityId entityId, List ts) { - Optional serverAddress = routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalTimeseriesUpdate(entityId, ts); - } else { - tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); - } - } - - private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - } - - private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - } - - private boolean isInTimeRange(Subscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - deviceSubscriptions.stream().filter(filter).forEach(s -> { - String sessionId = s.getWsSessionId(); - List subscriptionUpdate = f.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - if (s.isLocal()) { - updateSubscriptionState(sessionId, s, update); - wsService.sendWsMsg(sessionId, update); - } else { - tellRemoteSubUpdate(s.getServer(), sessionId, update); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { - log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); - update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); - } - - private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { - Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); - deviceSubscriptions.add(subscription); - Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - - private void cleanupLocalWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, true); - } - - private void cleanupRemoteWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, false); - } - - private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { - log.debug("[{}] Removing all subscriptions for particular session.", sessionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - int sessionSubscriptionSize = sessionSubscriptions.size(); - - for (Subscription subscription : sessionSubscriptions.values()) { - EntityId entityId = subscription.getEntityId(); - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - deviceSubscriptions.remove(subscription); - if (deviceSubscriptions.isEmpty()) { - subscriptionsByEntityId.remove(entityId); - } - } - subscriptionsByWsSessionId.remove(sessionId); - log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); - - if (localSession) { - notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); - } - } else { - log.debug("[{}] No subscriptions found!", sessionId); - } - } - - private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { - Set affectedServers = new HashSet<>(); - for (Subscription subscription : sessionSubscriptions.values()) { - if (subscription.getServer() != null) { - affectedServers.add(subscription.getServer()); - } - } - for (ServerAddress address : affectedServers) { - log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); - tellRemoteSessionClose(address, sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } - private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - if (subscription.isLocal() && subscription.getServer() != null) { - tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); - } - if (sessionSubscriptions.isEmpty()) { - log.debug("[{}] Removed last subscription for particular session.", sessionId); - subscriptionsByWsSessionId.remove(sessionId); - } else { - log.debug("[{}] Removed session subscription.", sessionId); - } - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - boolean result = deviceSubscriptions.remove(subscription); - if (result) { - if (deviceSubscriptions.size() == 0) { - log.debug("[{}] Removed last subscription for particular device.", sessionId); - subscriptionsByEntityId.remove(entityId); - } else { - log.debug("[{}] Removed device subscription.", sessionId); - } + private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); } else { - log.debug("[{}] Subscription not found!", sessionId); + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } } else { - log.debug("[{}] No device subscriptions found!", sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } @@ -617,161 +211,4 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } - - private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { - ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(sub.getSubscriptionId()); - builder.setTenantId(sub.getSub().getTenantId().getId().toString()); - builder.setEntityType(sub.getEntityId().getEntityType().name()); - builder.setEntityId(sub.getEntityId().getId().toString()); - builder.setType(sub.getType().name()); - builder.setAllKeys(sub.isAllKeys()); - if (sub.getScope() != null) { - builder.setScope(sub.getScope()); - } - sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( - ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { - ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(update.getSubscriptionId()); - builder.setErrorCode(update.getErrorCode()); - if (update.getErrorMsg() != null) { - builder.setErrorMsg(update.getErrorMsg()); - } - update.getData().entrySet().forEach( - e -> { - ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); - - dataBuilder.setKey(e.getKey()); - e.getValue().forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - - builder.addData(dataBuilder.build()); - } - ); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { - ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { - ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSessionClose(ServerAddress address, String sessionId) { - ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { - ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { - ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setTs(ts); - dataBuilder.setValueType(attr.getDataType().ordinal()); - switch (attr.getDataType()) { - case BOOLEAN: - Optional booleanValue = attr.getBooleanValue(); - booleanValue.ifPresent(dataBuilder::setBoolValue); - break; - case LONG: - Optional longValue = attr.getLongValue(); - longValue.ifPresent(dataBuilder::setLongValue); - break; - case DOUBLE: - Optional doubleValue = attr.getDoubleValue(); - doubleValue.ifPresent(dataBuilder::setDoubleValue); - break; - case JSON: - Optional jsonValue = attr.getJsonValue(); - jsonValue.ifPresent(dataBuilder::setJsonValue); - break; - case STRING: - Optional stringValue = attr.getStrValue(); - stringValue.ifPresent(dataBuilder::setStrValue); - break; - } - return dataBuilder; - } - - private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { - return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); - } - - private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { - return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); - } - - private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getValueType()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); - break; - } - return entry; - } - - private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - private Optional getSubscription(String sessionId, int subscriptionId) { - Subscription state = null; - Map subMap = subscriptionsByWsSessionId.get(sessionId); - if (subMap != null) { - state = subMap.get(subscriptionId); - } - return Optional.ofNullable(state); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 741efe6f8a..6afcaf97bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,12 +43,18 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; +import org.thingsboard.server.service.subscription.TbAttributeSubscription; +import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; @@ -57,7 +63,6 @@ import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -83,6 +88,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service +@TbCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { @@ -98,7 +104,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TelemetrySubscriptionService subscriptionManager; + private TbLocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -112,6 +118,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -127,9 +136,11 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; + private String serviceId; @PostConstruct public void initExecutor() { + serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -153,7 +164,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); processSessionClose(sessionRef); break; } @@ -334,8 +345,18 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState) + .scope(scope).build(); + subService.addSubscription(sub); } @Override @@ -421,8 +442,18 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState) + .scope(scope).build(); + subService.addSubscription(sub); } @Override @@ -494,8 +525,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -520,12 +559,19 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); - Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -544,9 +590,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); } else { - subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); + subService.cancelSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d77a0e50dc..d652a3f1c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,34 +15,13 @@ */ package org.thingsboard.server.service.telemetry; +import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { - void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); - - void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); - - void removeSubscription(String sessionId, int cmdId); - - void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); - - void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); - - void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java deleted file mode 100644 index 7f7db9430d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.telemetry.sub; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.TelemetryFeature; - -import java.util.Map; - -@Data -@AllArgsConstructor -public class Subscription { - - private final SubscriptionState sub; - private final boolean local; - private ServerAddress server; - private long startTime; - private long endTime; - - public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { - this(sub, local, server, 0L, 0L); - } - - public String getWsSessionId() { - return getSub().getWsSessionId(); - } - - public int getSubscriptionId() { - return getSub().getSubscriptionId(); - } - - public EntityId getEntityId() { - return getSub().getEntityId(); - } - - public TelemetryFeature getType() { - return getSub().getType(); - } - - public String getScope() { - return getSub().getScope(); - } - - public boolean isAllKeys() { - return getSub().isAllKeys(); - } - - public Map getKeyStates() { - return getSub().getKeyStates(); - } - - public void setKeyState(String key, long ts) { - getSub().getKeyStates().put(key, ts); - } - - @Override - public String toString() { - return "Subscription{" + - "sub=" + sub + - ", local=" + local + - ", server=" + server + - '}'; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 68aaca34c2..992dc26af9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { + return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java deleted file mode 100644 index 1aee532c93..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -@Service -@Slf4j -public class BaseRuleChainTransactionService implements RuleChainTransactionService { - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - - @Autowired - private DbCallbackExecutorService callbackExecutor; - - @Value("${actors.rule.transaction.queue_size}") - private int finalQueueSize; - @Value("${actors.rule.transaction.duration}") - private long duration; - - private final Lock transactionLock = new ReentrantLock(); - private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); - private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); - - private ExecutorService timeoutExecutor; - - @PostConstruct - public void init() { - timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); - executeOnTimeout(); - } - - @PreDestroy - public void destroy() { - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - } - - @Override - public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); - int queueSize = queue.size(); - if (queueSize >= finalQueueSize) { - log.trace("Queue has no space: {}", transactionTask); - executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); - } else { - addMsgToQueues(queue, transactionTask); - if (queueSize == 0) { - executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); - } else { - log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); - } - } - } finally { - transactionLock.unlock(); - } - } - - @Override - public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); - if (address.isPresent()) { - sendTransactionEventToRemoteServer(msg, address.get()); - executeOnSuccess(onSuccess, msg); - } else { - endLocalTransaction(msg, onSuccess, onFailure); - } - } - - @Override - public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data), msg -> { - }, error -> { - }); - } - - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { - queue.offer(transactionTask); - timeoutQueue.offer(transactionTask); - log.trace("Added msg to queue, size: [{}]", queue.size()); - } - - private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask currentTransactionTask = queue.peek(); - if (currentTransactionTask != null) { - if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { - currentTransactionTask.setCompleted(true); - queue.poll(); - log.trace("Removed msg from queue, size [{}]", queue.size()); - - executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); - executeOnSuccess(onSuccess, msg); - - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } else { - log.trace("Task has expired!"); - executeOnFailure(onFailure, "Task has expired!"); - } - } else { - log.trace("Queue is empty, previous task has expired!"); - executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); - } - } finally { - transactionLock.unlock(); - } - } - - private void executeOnTimeout() { - timeoutExecutor.submit(() -> { - while (true) { - TbTransactionTask transactionTask = timeoutQueue.peek(); - if (transactionTask != null) { - long sleepDuration = 0L; - transactionLock.lock(); - try { - if (transactionTask.isCompleted()) { - timeoutQueue.poll(); - } else { - long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); - if (expIn < 0) { - log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - timeoutQueue.poll(); - executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); - - BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); - if (queue != null) { - queue.poll(); - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } - } else { - sleepDuration = Math.min(expIn, duration); - } - } - } finally { - transactionLock.unlock(); - } - if (sleepDuration > 0L) { - try { - log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - TimeUnit.MILLISECONDS.sleep(sleepDuration); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } else { - try { - log.trace("Queue is empty, waiting for tasks!"); - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } - }); - } - - private void executeOnFailure(Consumer onFailure, String exception) { - executeCallback(() -> { - onFailure.accept(new RuntimeException(exception)); - return null; - }); - } - - private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { - executeCallback(() -> { - onSuccess.accept(tbMsg); - return null; - }); - } - - private void executeCallback(Callable task) { - callbackExecutor.executeAsync(task); - } - - private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { - log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java deleted file mode 100644 index c86f882ff7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -@Data -@AllArgsConstructor -public final class TbTransactionTask { - - private final TbMsg msg; - private final Consumer onStart; - private final Consumer onEnd; - private final Consumer onFailure; - private final long expirationTime; - - private boolean isCompleted; - - public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { - this.msg = msg; - this.onStart = onStart; - this.onEnd = onEnd; - this.onFailure = onFailure; - this.expirationTime = expirationTime; - this.isCompleted = false; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java new file mode 100644 index 0000000000..c9e37e6791 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; + +@Slf4j +@Service +@TbCoreComponent +public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + + private final PartitionService partitionService; + private final TbQueueProducer> tbTransportProducer; + + public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { + this.partitionService = partitionService; + this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); + } + + @Override + public void process(String nodeId, ToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); + tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private static class QueueCallbackAdaptor implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 8e820fbffe..7456ca7662 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -33,14 +34,19 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -52,10 +58,15 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -public class LocalTransportApiService implements TransportApiService { +@TbCoreComponent +public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); + //TODO: Constructor dependencies; + @Autowired + private TenantService tenantService; + @Autowired private DeviceService deviceService; @@ -74,17 +85,20 @@ public class LocalTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { + public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { + TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); + return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); + return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); + return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } - return getEmptyTransportApiResponseFuture(); + return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -125,6 +139,13 @@ public class LocalTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } + private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); + ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); + return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() + .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) + .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); + } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java deleted file mode 100644 index 32ff77877d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.service.AbstractTransportService; -import org.thingsboard.server.dao.device.ClaimDevicesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") -public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { - - @Autowired - private TransportApiService transportApiService; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - @Autowired - private ClaimDevicesService claimDevicesService; - - @PostConstruct - public void init() { - super.init(); - } - - @PreDestroy - public void destroy() { - super.destroy(); - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); - } - - @Override - public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); - - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - callback.onSuccess(null); - } else { - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); - DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), - callback::onSuccess, callback::onError); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - processToTransportMsg(msg); - if (onSuccess != null) { - onSuccess.run(); - } - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - if (callback != null) { - callback.onSuccess(null); - } - } - - private Consumer getThrowableConsumer(TransportServiceCallback callback) { - return e -> { - if (callback != null) { - callback.onError(e); - } - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java deleted file mode 100644 index 7aa438c64f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.BlockingBucket; -import io.github.bucket4j.Bucket4j; -import io.github.bucket4j.local.LocalBucket; -import io.github.bucket4j.local.LocalBucketBuilder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 09.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService implements RuleEngineTransportService { - - @Value("${transport.remote.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${transport.remote.notifications.topic}") - private String notificationsTopic; - @Value("${transport.remote.rule_engine.poll_interval}") - private int pollDuration; - @Value("${transport.remote.rule_engine.auto_commit_interval}") - private int autoCommitInterval; - - @Value("${transport.remote.rule_engine.poll_records_pack_size}") - private int pollRecordsPackSize; - @Value("${transport.remote.rule_engine.max_poll_records_per_second}") - private long pollRecordsPerSecond; - @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") - private long pollRecordsPerMinute; - @Value("${transport.remote.rule_engine.stats.enabled:false}") - private boolean statsEnabled; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - - private TBKafkaConsumerTemplate ruleEngineConsumer; - private TBKafkaProducerTemplate notificationsProducer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); - - private volatile boolean stopped = false; - - private final RuleEngineStats stats = new RuleEngineStats(); - - @PostConstruct - public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); - notificationsProducerBuilder.settings(kafkaSettings); - notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); - notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); - - notificationsProducer = notificationsProducerBuilder.build(); - notificationsProducer.init(); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); - ruleEngineConsumerBuilder.settings(kafkaSettings); - ruleEngineConsumerBuilder.topic(ruleEngineTopic); - ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - ruleEngineConsumerBuilder.groupId("tb-node"); - ruleEngineConsumerBuilder.autoCommit(true); - ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); - ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); - ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); - - ruleEngineConsumer = ruleEngineConsumerBuilder.build(); - ruleEngineConsumer.subscribe(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - LocalBucketBuilder builder = Bucket4j.builder(); - builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); - builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); - LocalBucket pollRateBucket = builder.build(); - BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); - int recordsCount = records.count(); - if (recordsCount > 0) { - while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { - log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - log.trace("Processing {} records", recordsCount); - } - records.forEach(record -> { - try { - ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - log.trace("Pushing message to local server: {}", toDeviceActorMsg); - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - } - - @PreDestroy - public void destroy() { - stopped = true; - if (ruleEngineConsumer != null) { - ruleEngineConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - private static class QueueCallbackAdaptor implements Callback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - if (onSuccess != null) { - onSuccess.run(); - } - } else { - if (onFailure != null) { - onFailure.accept(exception); - } - } - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java deleted file mode 100644 index 0c9efc7d1c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.*; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 05.10.18. - */ -@Slf4j -@Component -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteTransportApiService { - - @Value("${transport.remote.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${transport.remote.transport_api.max_pending_requests}") - private int maxPendingRequests; - @Value("${transport.remote.transport_api.request_timeout}") - private long requestTimeout; - @Value("${transport.remote.transport_api.request_poll_interval}") - private int responsePollDuration; - @Value("${transport.remote.transport_api.request_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TransportApiService transportApiService; - - private ExecutorService transportCallbackExecutor; - - private TbKafkaResponseTemplate transportApiTemplate; - - @PostConstruct - public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(100); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); - responseBuilder.encoder(new TransportApiResponseEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.topic(transportApiRequestsTopic); - requestBuilder.clientId(nodeIdProvider.getNodeId()); - requestBuilder.groupId("tb-node"); - requestBuilder.autoCommit(true); - requestBuilder.autoCommitIntervalMs(autoCommitInterval); - requestBuilder.decoder(new TransportApiRequestDecoder()); - - TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder - builder = TbKafkaResponseTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.requestTimeout(requestTimeout); - builder.pollInterval(responsePollDuration); - builder.executor(transportCallbackExecutor); - builder.handler(transportApiService); - transportApiTemplate = builder.build(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(); - } - - @PreDestroy - public void destroy() { - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java rename to application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 9c3f5634bf..e0d5e2121a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -16,14 +16,13 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToTransportMsgEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(ToTransportMsg value) { - return value.toByteArray(); - } +import java.util.function.Consumer; + +public interface TbCoreToTransportService { + + void process(String nodeId, ToTransportMsg msg); + + void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java new file mode 100644 index 0000000000..566bf99cd3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 05.10.18. + */ +@Slf4j +@Service +@TbCoreComponent +public class TbCoreTransportApiService { + + private final TbCoreQueueFactory tbCoreQueueFactory; + private final TransportApiService transportApiService; + + @Value("${queue.transport_api.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.transport_api.max_requests_timeout:10000}") + private long requestTimeout; + @Value("${queue.transport_api.request_poll_interval:25}") + private int responsePollDuration; + @Value("${queue.transport_api.max_callback_threads:100}") + private int maxCallbackThreads; + + private ExecutorService transportCallbackExecutor; + private TbQueueResponseTemplate, + TbProtoQueueMsg> transportApiTemplate; + + public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { + this.tbCoreQueueFactory = tbCoreQueueFactory; + this.transportApiService = transportApiService; + } + + @PostConstruct + public void init() { + this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); + TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); + + DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); + builder.requestTemplate(consumer); + builder.responseTemplate(producer); + builder.maxPendingRequests(maxPendingRequests); + builder.requestTimeout(requestTimeout); + builder.pollInterval(responsePollDuration); + builder.executor(transportCallbackExecutor); + builder.handler(transportApiService); + transportApiTemplate = builder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + transportApiTemplate.init(transportApiService); + } + + @PreDestroy + public void destroy() { + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java deleted file mode 100644 index ac1c5965f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; - -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiRequestDecoder implements TbKafkaDecoder { - @Override - public TransportApiRequestMsg decode(byte[] data) throws IOException { - return TransportApiRequestMsg.parseFrom(data); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java deleted file mode 100644 index 35f8a7b1aa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.kafka.TbKafkaEncoder; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiResponseEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(TransportApiResponseMsg value) { - return value.toByteArray(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 2964934313..c9edfa9331 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.kafka.TbKafkaHandler; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbKafkaHandler { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 34abac72bb..ba4e7baab1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -36,9 +37,11 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; + private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { this.msg = msg; + this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 2399450d86..61d81ae0b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlDao; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService { - - @Value("${sql.ttl.ts_key_value_ttl}") - protected long systemTtl; - @Value("${sql.ttl.enabled}") - private boolean ttlTaskExecutionEnabled; +@Slf4j +@PsqlDao +public abstract class AbstractCleanUpService { @Value("${spring.datasource.url}") - private String dbUrl; + protected String dbUrl; @Value("${spring.datasource.username}") - private String dbUserName; + protected String dbUserName; @Value("${spring.datasource.password}") - private String dbPassword; - - @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - protected abstract void doCleanUp(Connection connection); + protected String dbPassword; protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService { return removed; } - private void getWarnings(Statement statement) throws SQLException { + protected void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService { } } -} \ No newline at end of file + protected abstract void doCleanUp(Connection connection); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java new file mode 100644 index 0000000000..a608ca257b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.events; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlDao +@Slf4j +@Service +public class EventsCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.events.events_ttl}") + private long ttl; + + @Value("${sql.ttl.events.debug_events_ttl}") + private long debugTtl; + + @Value("${sql.ttl.events.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + @Override + protected void doCleanUp(Connection connection) { + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); + log.info("Total events removed by TTL: [{}]", totalEventsRemoved); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java new file mode 100644 index 0000000000..75b07b9176 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlTsAnyDao +@Slf4j +public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.ts.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.ts.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index ab344dc518..cd403ee3b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index 1dbdb4ad55..f5898b9b20 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 1b8173c424..4b7f0f6366 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,6 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service +@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto deleted file mode 100644 index cfacc66121..0000000000 --- a/application/src/main/proto/cluster.proto +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package cluster; - -option java_package = "org.thingsboard.server.gen.cluster"; -option java_outer_classname = "ClusterAPIProtos"; - -service ClusterRpcService { - rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} -} - -message ClusterMessage { - MessageType messageType = 1; - MessageMataInfo messageMetaInfo = 2; - ServerAddress serverAddress = 3; - bytes payload = 4; -} - -message ServerAddress { - string host = 1; - int32 port = 2; -} - -message MessageMataInfo { - string payloadMetaInfo = 1; - repeated string tags = 2; -} - -enum MessageType { - - //Cluster control messages - RPC_SESSION_CREATE_REQUEST_MSG = 0; - TO_ALL_NODES_MSG = 1; - RPC_SESSION_TELL_MSG = 2; - RPC_BROADCAST_MSG = 3; - CONNECT_RPC_MESSAGE =4; - - CLUSTER_ACTOR_MESSAGE = 5; - // Messages related to TelemetrySubscriptionService - CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; - CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; - CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; - CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; - CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; - CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; - CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; - - CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; - CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; -} - -// Messages related to CLUSTER_TELEMETRY_MESSAGE -message SubscriptionProto { - string sessionId = 1; - int32 subscriptionId = 2; - string entityType = 3; - string tenantId = 4; - string entityId = 5; - string type = 6; - bool allKeys = 7; - repeated SubscriptionKetStateProto keyStates = 8; - string scope = 9; -} - -message SubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated SubscriptionUpdateValueListProto data = 5; -} - -message AttributeUpdateProto { - string entityType = 1; - string entityId = 2; - string scope = 3; - repeated KeyValueProto data = 4; -} - -message TimeseriesUpdateProto { - string entityType = 1; - string entityId = 2; - repeated KeyValueProto data = 4; -} - -message SessionCloseProto { - string sessionId = 1; -} - -message SubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message SubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message SubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} - -message KeyValueProto { - string key = 1; - int64 ts = 2; - int32 valueType = 3; - string strValue = 4; - int64 longValue = 5; - double doubleValue = 6; - bool boolValue = 7; - string jsonValue = 8; -} - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e147325f42..614f85ba0e 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,6 +28,9 @@ + + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8dc7089bbe..c896b3fc85 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -70,21 +70,7 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# RPC connection parameters. Used only in cluster mode only. -rpc: - bind_host: "${RPC_HOST:localhost}" - bind_port: "${RPC_PORT:9001}" - -# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" - # Name of hash function used for consistent hash ring. - hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" - # Amount of virtual nodes in consistent hash ring. - vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" - # Queue partition id for current node - partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -215,14 +201,18 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - enabled: "${SQL_TTL_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds - ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds + ts: + enabled: "${SQL_TTL_TS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds + events: + enabled: "${SQL_TTL_EVENTS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: - cluster: - grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -265,8 +255,6 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -346,19 +334,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -387,7 +375,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" # Audit log parameters audit-log: @@ -427,32 +415,11 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -462,55 +429,27 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: - type: "${TRANSPORT_TYPE:local}" # local or remote - remote: - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" - max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" - max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" - stats: - enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -523,6 +462,8 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -566,7 +507,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:Thingsboard REST API}" + title: "${SWAGGER_TITLE:ThingsBoard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -576,3 +517,160 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" + +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index bed5689ceb..1e445adf3c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -115,9 +116,7 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), - MediaType.APPLICATION_JSON.getSubtype(), - Charset.forName("utf8")); + protected MediaType contentType = MediaType.APPLICATION_JSON; protected MockMvc mockMvc; @@ -200,6 +199,7 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); + log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 23e23c31fe..c74c83dfe8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/"+savedAsset.getId().getId().toString()) + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/"+savedAsset.getId().getId().toString()) + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -269,6 +271,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); + Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -279,10 +283,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -290,10 +294,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -334,7 +338,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -345,7 +349,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -361,10 +365,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -373,10 +377,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -417,7 +421,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -428,7 +432,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -447,9 +451,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -483,10 +487,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -496,10 +500,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -574,10 +578,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -588,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 4c7667a7c2..34c39bdccb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -24,6 +25,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CustomerId; @@ -42,6 +44,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -51,6 +54,7 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; +@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -417,12 +421,22 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); - + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); + client.disconnect(); + } + + private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { + long start = System.currentTimeMillis(); + while (!client.isConnected()) { + Thread.sleep(100); + if (start + ms < System.currentTimeMillis()) { + throw new RuntimeException("Client is not connected!"); + } + } } private Set getTelemetryKeys(String type, String id) throws Exception { @@ -449,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - + client.disconnect(); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 8dc0acff57..347eaee7eb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.controller; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index c43637e05f..a9bb113a60 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,11 +16,13 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -41,4 +43,9 @@ public class MqttNoSqlTestSuite { Arrays.asList( new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 2863589ba1..70de3b27ca 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.mqtt; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -32,4 +34,9 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 5ba4dfcbae..9ddb2c81d9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; + private static final AtomicInteger atomicInteger = new AtomicInteger(2); + @Before public void beforeTest() throws Exception { @@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -109,6 +112,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -128,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -137,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -165,7 +170,9 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + Thread.sleep(2000); + + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -183,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -192,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f1580c65c7..47e9537ef9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,8 +79,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(1000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + Thread.sleep(2000); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -97,20 +97,25 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - @Test + +// @Test - Unstable public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); - client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); + client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"value\"}"; - String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); + String payload = "{\"key\":\"uniqueValue\"}"; +// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. +// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) +// MqttClient <- SUB_ACK <- Transport + Thread.sleep(5000); + doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -120,8 +125,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private Integer qoS; - private String payload; + private volatile Integer qoS; + private volatile String payload; String getPayload() { return payload; @@ -138,6 +143,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { + log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 5f930821f7..83ac5f7703 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.rules; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index cfac1ea66b..97e6e59969 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,19 +15,21 @@ */ package org.thingsboard.server.rules.flow; -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; -import lombok.Data; +import akka.actor.ActorRef; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; @@ -37,17 +39,14 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.rule.RuleChainService; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -62,7 +61,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -149,15 +148,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -264,15 +260,13 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 1d35168e78..088f9390ff 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,15 +15,17 @@ */ package org.thingsboard.server.rules.lifecycle; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -38,13 +40,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -61,7 +63,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -136,16 +138,13 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", - null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java deleted file mode 100644 index d717169048..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.datastax.driver.core.utils.UUIDs; -import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.UUIDConverter; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class ConsistentClusterRoutingServiceTest { - - private ConsistentClusterRoutingService clusterRoutingService; - - private DiscoveryService discoveryService; - - private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 1024*4; - private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); - - - @Before - public void setup() throws Exception { - discoveryService = mock(DiscoveryService.class); - clusterRoutingService = new ConsistentClusterRoutingService(); - ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); - List otherServers = new ArrayList<>(); - for (int i = 1; i < 30; i++) { - otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); - } - when(discoveryService.getOtherServers()).thenReturn(otherServers); - clusterRoutingService.init(); - } - - @Test - public void testDispersionOnMillionDevices() { - List devices = new ArrayList<>(); - for (int i = 0; i < 1000000; i++) { - devices.add(new DeviceId(UUIDs.timeBased())); - } - - testDevicesDispersion(devices); - } - - private void testDevicesDispersion(List devices) { - long start = System.currentTimeMillis(); - Map map = new HashMap<>(); - for (DeviceId deviceId : devices) { - ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); - map.put(address, map.getOrDefault(address, 0) + 1); - } - - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); - long end = System.currentTimeMillis(); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); - - for (Map.Entry entry : data) { -// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); - } - - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java new file mode 100644 index 0000000000..3f7619ed1e --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.datastax.driver.core.utils.UUIDs; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ConsistentHashParitionServiceTest { + + public static final int ITERATIONS = 1000000; + private ConsistentHashPartitionService clusterRoutingService; + + private TbServiceInfoProvider discoveryService; + private TenantRoutingInfoService routingInfoService; + private ApplicationEventPublisher applicationEventPublisher; + + private String hashFunctionName = "murmur3_128"; + private Integer virtualNodesSize = 16; + + + @Before + public void setup() throws Exception { + discoveryService = mock(TbServiceInfoProvider.class); + applicationEventPublisher = mock(ApplicationEventPublisher.class); + routingInfoService = mock(TenantRoutingInfoService.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); + ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); + + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); + ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); + TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96.1.1") + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build(); +// when(discoveryService.getServiceInfo()).thenReturn(currentServer); + List otherServers = new ArrayList<>(); + for (int i = 1; i < 30; i++) { + otherServers.add(TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96." + i * 2 + "." + i) + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build()); + } + clusterRoutingService.init(); + clusterRoutingService.recalculatePartitions(currentServer, otherServers); + } + + @Test + public void testDispersionOnMillionDevices() { + List devices = new ArrayList<>(); + for (int i = 0; i < ITERATIONS; i++) { + devices.add(new DeviceId(UUIDs.timeBased())); + } + testDevicesDispersion(devices); + } + + private void testDevicesDispersion(List devices) { + long start = System.currentTimeMillis(); + Map map = new HashMap<>(); + for (DeviceId deviceId : devices) { + TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); + Integer partition = address.getPartition().get(); + map.put(partition, map.getOrDefault(partition, 0) + 1); + } + + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + long end = System.currentTimeMillis(); + double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); + + for (Map.Entry entry : data) { + System.out.println(entry.getKey() + ": " + entry.getValue()); + } + + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java new file mode 100644 index 0000000000..43cd62ea97 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class TbMsgPackProcessingContextTest { + + @Test + public void testHighConcurrencyCase() throws InterruptedException { + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + int msgCount = 1000; + int parallelCount = 5; + ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); + try { + ConcurrentMap> messages = new ConcurrentHashMap<>(); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + when(strategyMock.getPendingMap()).thenReturn(messages); + TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); + for (UUID uuid : messages.keySet()) { + for (int i = 0; i < parallelCount; i++) { + executorService.submit(() -> context.onSuccess(uuid)); + } + } + Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); + Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); + } finally { + executorService.shutdownNow(); + } + } +} diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 82b3df9736..62881bd5af 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,6 +24,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -78,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -94,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -112,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -126,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -147,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -169,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 1065fbf6d3..8a89de5dff 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,4 +49,9 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return 100000; + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java new file mode 100644 index 0000000000..c4182db3ee --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.system; + +import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.runner.RunWith; +import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; + +import java.util.Arrays; + +/** + * @author Andrew Shvayka + */ +@RunWith(ClasspathSuite.class) +@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.system.*NoSqlTest"}) +public class SystemNoSqlTestSuite { + + @ClassRule + public static CustomCassandraCQLUnit cassandraUnit = + new CustomCassandraCQLUnit( + Arrays.asList( + new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), + new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), + new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), + "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } +} diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index b12d513ce0..6060a2cc2b 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.system; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,4 +37,9 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index bf5a494afb..298800a2f0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index 96a36e1f65..a41dd2ee21 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index 20954b8ac2..bcf1222988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index ebd0ef7d28..7b10e0cf03 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index ef73ec8f90..a888926bd0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index f2a8800032..a215a16b29 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index c3d607f8fe..76db7920a6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.util; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +@Retention(RetentionPolicy.RUNTIME) public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9a43c530a2..9be3321988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index abba0e985b..0b665c5695 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 20745d790c..0541a47879 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index b1f6a967b6..766d405e25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,6 +29,8 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; + private boolean isolatedTbCore; + private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -72,6 +74,22 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchText() { return getTitle(); @@ -84,6 +102,10 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); + builder.append(", isolatedTbCore="); + builder.append(isolatedTbCore); + builder.append(", isolatedTbRuleEngine="); + builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index a43d95975e..9c23a60c28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,6 +23,7 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java index 532842ae75..691d89cafd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java @@ -13,14 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.alarm; +package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index c367355cb0..11db1da1a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 5639f98d01..2993d81154 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,6 +32,10 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } + public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { + this(kv, lastUpdateTs); + } + @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 04b0cae56b..0345355aa3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; + /** * Created by ashvayka on 15.03.18. */ @@ -24,17 +27,12 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} + * See {@link PartitionChangeMsg} */ - CLUSTER_EVENT_MSG, + PARTITION_CHANGE_MSG, APP_INIT_MSG, - /** - * All messages, could be send to cluster - */ - SEND_TO_CLUSTER_MSG, - /** * ADDED/UPDATED/DELETED events for main entities. * @@ -43,11 +41,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages from the REST API/SERVICE layer to the new rule engine. + * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. * - * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} + * See {@link QueueToRuleEngineMsg} */ - SERVICE_TO_RULE_ENGINE_MSG, + QUEUE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. @@ -91,12 +89,9 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, - DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, - /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ - DEVICE_ACTOR_TO_RULE_ENGINE_MSG, SESSION_TIMEOUT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 4e7090f934..3edf4e5061 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -15,24 +15,27 @@ */ package org.thingsboard.server.common.msg; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.UUID; /** * Created by ashvayka on 13.01.18. */ @Data -@AllArgsConstructor +@Builder +@Slf4j public final class TbMsg implements Serializable { private final UUID id; @@ -41,30 +44,61 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; - private final TbMsgTransactionData transactionData; - - //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - private final long clusterPartition; + //This field is not serialized because we use queues and there is no need to do it + private final TbMsgCallback callback; + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + } + + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + } + + public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; + this.dataType = dataType; this.data = data; - this.dataType = TbMsgDataType.JSON; - this.transactionData = new TbMsgTransactionData(id, originator); this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; - this.clusterPartition = clusterPartition; + if (callback != null) { + this.callback = callback; + } else { + log.warn("[{}] Created message with empty callback: {}", originator, type); + this.callback = TbMsgCallback.EMPTY; + } } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); + public static ByteString toByteString(TbMsg msg) { + return ByteString.copyFrom(toByteArray(msg)); } public static byte[] toByteArray(TbMsg msg) { @@ -89,52 +123,37 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } - TbMsgTransactionData transactionData = msg.getTransactionData(); - if (transactionData != null) { - MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); - transactionBuilder.setId(transactionData.getTransactionId().toString()); - transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); - transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); - transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); - builder.setTransactionData(transactionBuilder.build()); - } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); - } - public static ByteBuffer toBytes(TbMsg msg) { - return ByteBuffer.wrap(toByteArray(msg)); - } - - public static TbMsg fromBytes(byte[] data) { - return fromBytes(ByteBuffer.wrap(data)); - } - - public static TbMsg fromBytes(ByteBuffer buffer) { + public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); - EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), - new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); - TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; + if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { + ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); + public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); } + public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index a52c237f87..d05948206c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java similarity index 71% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index 56f928a1c8..0e8d66aa63 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.common.msg.queue; import lombok.Data; +import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import java.util.Set; + /** * @author Andrew Shvayka */ @Data -public final class ClusterEventMsg implements TbActorMsg { +public final class PartitionChangeMsg implements TbActorMsg { - private final ServerAddress serverAddress; - private final boolean added; + @Getter + private final ServiceQueueKey serviceQueueKey; + @Getter + private final Set partitions; @Override public MsgType getMsgType() { - return MsgType.CLUSTER_EVENT_MSG; + return MsgType.PARTITION_CHANGE_MSG; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java similarity index 73% rename from common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index a717af85a4..fca1c51995 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.system; +package org.thingsboard.server.common.msg.queue; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -22,18 +22,25 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; +import java.util.Set; /** * Created by ashvayka on 15.03.18. */ @Data -public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { +public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; + private final Set relationTypes; + private final String failureMessage; @Override public MsgType getMsgType() { - return MsgType.SERVICE_TO_RULE_ENGINE_MSG; + return MsgType.QUEUE_TO_RULE_ENGINE_MSG; + } + + public boolean isTellNext() { + return relationTypes != null && !relationTypes.isEmpty(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java new file mode 100644 index 0000000000..dd720251bb --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuleEngineException extends Exception { + protected static final ObjectMapper mapper = new ObjectMapper(); + + public RuleEngineException(String message) { + super(message != null ? message : "Unknown"); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java new file mode 100644 index 0000000000..3f437dd1d7 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.rule.RuleNode; + +@Slf4j +public class RuleNodeException extends RuleEngineException { + @Getter + private final String ruleChainName; + @Getter + private final String ruleNodeName; + @Getter + private final RuleChainId ruleChainId; + @Getter + private final RuleNodeId ruleNodeId; + + public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { + super(message); + this.ruleChainName = ruleChainName; + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode() + .put("ruleNodeId", ruleNodeId.toString()) + .put("ruleChainId", ruleChainId.toString()) + .put("ruleNodeName", ruleNodeName) + .put("ruleChainName", ruleChainName) + .put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java new file mode 100644 index 0000000000..be08f054b6 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.ToString; + +import java.util.Objects; + +@ToString +public class ServiceQueue { + + public static final String MAIN = "Main"; + + private final ServiceType type; + private final String queue; + + public ServiceQueue(ServiceType type) { + this.type = type; + this.queue = MAIN; + } + + public ServiceQueue(ServiceType type, String queue) { + this.type = type; + this.queue = queue; + } + + public ServiceType getType() { + return type; + } + + public String getQueue() { + return queue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueue that = (ServiceQueue) o; + return type == that.type && + queue.equals(that.queue); + } + + @Override + public int hashCode() { + return Objects.hash(type, queue); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java similarity index 50% rename from application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java index 570112dab3..f4bfefd878 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java @@ -13,35 +13,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.rpc; +package org.thingsboard.server.common.msg.queue; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -/** - * Created by ashvayka on 16.04.18. - */ +import java.util.Objects; + @ToString -@RequiredArgsConstructor -public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { +public class ServiceQueueKey { + @Getter + private final ServiceQueue serviceQueue; @Getter private final TenantId tenantId; - @Getter - private final DeviceId deviceId; + public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { + this.serviceQueue = serviceQueue; + this.tenantId = tenantId; + } - @Getter - private final ToServerRpcResponseMsg msg; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueueKey that = (ServiceQueueKey) o; + return serviceQueue.equals(that.serviceQueue) && + Objects.equals(tenantId, that.tenantId); + } @Override - public MsgType getMsgType() { - return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; + public int hashCode() { + return Objects.hash(serviceQueue, tenantId); + } + + public ServiceType getServiceType() { + return serviceQueue.getType(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java new file mode 100644 index 0000000000..0ca776126d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +public enum ServiceType { + + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + + public static ServiceType of(String serviceType) { + return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index d0ba27814c..4c25f27a3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -13,21 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.common.msg.queue; -import java.util.List; +public interface TbCallback { -/** - * @author Andrew Shvayka - */ -public interface DiscoveryService { + TbCallback EMPTY = new TbCallback() { + + @Override + public void onSuccess() { + + } - void publishCurrentServer(); + @Override + public void onFailure(Throwable t) { - void unpublishCurrentServer(); + } + }; - ServerInstance getCurrentServer(); + void onSuccess(); - List getOtherServers(); + void onFailure(Throwable t); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java similarity index 63% rename from common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 5d57514275..9e8d5ae6b8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -13,18 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg; +package org.thingsboard.server.common.msg.queue; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; +public interface TbMsgCallback { -import java.io.Serializable; -import java.util.UUID; + TbMsgCallback EMPTY = new TbMsgCallback() { -@Data -public final class TbMsgTransactionData implements Serializable { + @Override + public void onSuccess() { - private final UUID transactionId; - private final EntityId originatorId; + } + + @Override + public void onFailure(RuleEngineException e) { + + } + }; + + void onSuccess(); + + void onFailure(RuleEngineException e); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java new file mode 100644 index 0000000000..bf0fffe404 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; +import java.util.Optional; + +@ToString +public class TopicPartitionInfo { + + private final String topic; + private final TenantId tenantId; + private final Integer partition; + @Getter + private final String fullTopicName; + @Getter + private final boolean myPartition; + + @Builder + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this.topic = topic; + this.tenantId = tenantId; + this.partition = partition; + this.myPartition = myPartition; + String tmp = topic; + if (tenantId != null) { + tmp += "." + tenantId.getId().toString(); + } + if (partition != null) { + tmp += "." + partition; + } + this.fullTopicName = tmp; + } + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + + public Optional getPartition() { + return Optional.ofNullable(partition); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfo that = (TopicPartitionInfo) o; + return topic.equals(that.topic) && + Objects.equals(tenantId, that.tenantId) && + Objects.equals(partition, that.partition) && + fullTopicName.equals(that.fullTopicName); + } + + @Override + public int hashCode() { + return Objects.hash(fullTopicName); + } +} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 03dc17b5c9..737d5f6f57 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,13 +23,6 @@ message TbMsgMetaDataProto { map data = 1; } -message TbMsgTransactionDataProto { - string id = 1; - string entityType = 2; - int64 entityIdMSB = 3; - int64 entityIdLSB = 4; -} - message TbMsgProto { string id = 1; string type = 2; @@ -46,7 +39,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - TbMsgTransactionDataProto transactionData = 12; + //Transaction Data (12) was removed in 2.5 int32 dataType = 13; string data = 14; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 24881a0e3f..b4a223cac0 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + util + org.thingsboard.common message @@ -48,6 +52,22 @@ org.apache.kafka kafka-clients + + com.amazonaws + aws-java-sdk-sqs + + + com.google.cloud + google-cloud-pubsub + + + com.microsoft.azure + azure-servicebus + + + com.rabbitmq + amqp-client + org.springframework spring-context-support @@ -84,6 +104,14 @@ ch.qos.logback logback-classic + + com.google.protobuf + protobuf-java + + + org.apache.curator + curator-recipes + junit junit @@ -96,4 +124,13 @@ + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java deleted file mode 100644 index 0fe86e030b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import org.apache.kafka.clients.admin.*; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.KafkaFuture; - -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * Created by ashvayka on 24.09.18. - */ -public class TBKafkaAdmin { - - AdminClient client; - - public TBKafkaAdmin(TbKafkaSettings settings) { - client = AdminClient.create(settings.toProps()); - } - - public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { - synchronized (this) { - long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); - while (!topicExists(topic)) { - long waitMs = timeoutExpiredMs - System.currentTimeMillis(); - if (waitMs <= 0) { - throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); - } else { - wait(1000); - } - } - } - } - - public CreateTopicsResult createTopic(NewTopic topic){ - return client.createTopics(Collections.singletonList(topic)); - } - - private boolean topicExists(String topic) throws InterruptedException { - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - try { - topicDescriptionFuture.get(); - return true; - } catch (ExecutionException e) { - return false; - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java deleted file mode 100644 index 73f8cc16c7..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.Getter; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; - -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.UUID; - -/** - * Created by ashvayka on 24.09.18. - */ -public class TBKafkaConsumerTemplate { - - private final KafkaConsumer consumer; - private final TbKafkaDecoder decoder; - - @Builder.Default - private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); - - @Getter - private final String topic; - - @Builder - private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - TbKafkaRequestIdExtractor requestIdExtractor, - String clientId, String groupId, String topic, - boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords) { - Properties props = settings.toProps(); - props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - if (groupId != null) { - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); - } - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (maxPollRecords > 0) { - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); - } - this.consumer = new KafkaConsumer<>(props); - this.decoder = decoder; - this.requestIdExtractor = requestIdExtractor; - this.topic = topic; - } - - public void subscribe() { - consumer.subscribe(Collections.singletonList(topic)); - } - - public void unsubscribe() { - consumer.unsubscribe(); - } - - public ConsumerRecords poll(Duration duration) { - return consumer.poll(duration); - } - - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(record.value()); - } - - public UUID extractRequestId(T value) { - return requestIdExtractor.extractRequestId(value); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java deleted file mode 100644 index 8722c1d64e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaFuture; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.springframework.util.StringUtils; - -import java.util.List; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TBKafkaProducerTemplate { - - private final KafkaProducer producer; - private final TbKafkaEncoder encoder; - - private final TbKafkaPartitioner partitioner; - private ConcurrentMap> partitionInfoMap; - @Getter - private final String defaultTopic; - - @Getter - private final TbKafkaSettings settings; - - @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, - TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { - Properties props = settings.toProps(); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - if (!StringUtils.isEmpty(clientId)) { - props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - } - this.settings = settings; - this.producer = new KafkaProducer<>(props); - this.encoder = encoder; - this.partitioner = partitioner; - this.defaultTopic = defaultTopic; - } - - public void init() { - this.partitionInfoMap = new ConcurrentHashMap<>(); - if (!StringUtils.isEmpty(defaultTopic)) { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); - admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); - log.info("[{}] Topic exists.", defaultTopic); - } catch (Exception e) { - log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); - throw new RuntimeException(e); - } - //Maybe this should not be cached, but we don't plan to change size of partitions - this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); - } - } - - public Future send(String key, T value, Callback callback) { - return send(key, value, null, callback); - } - - public Future send(String key, T value, Iterable
headers, Callback callback) { - return send(key, value, null, headers, callback); - } - - public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { - if (!StringUtils.isEmpty(this.defaultTopic)) { - return send(this.defaultTopic, key, value, timestamp, headers, callback); - } else { - throw new RuntimeException("Failed to send message! Default topic is not specified!"); - } - } - - public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { - return send(topic, key, value, null, headers, callback); - } - - public Future send(String topic, String key, T value, Callback callback) { - return send(topic, key, value, null, null, callback); - } - - public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { - byte[] data = encoder.encode(value); - ProducerRecord record; - Integer partition = getPartition(topic, key, value, data); - record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); - return producer.send(record, callback); - } - - private Integer getPartition(String topic, String key, T value, byte[] data) { - if (partitioner == null) { - return null; - } else { - return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java deleted file mode 100644 index 864182a552..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.time.Duration; -import java.util.Collections; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaConsumerTemplate requestTemplate; - private final TBKafkaProducerTemplate responseTemplate; - private final TbKafkaHandler handler; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, - TBKafkaProducerTemplate responseTemplate, - TbKafkaHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.handler = handler; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - public void init() { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); - requests.forEach(request -> { - Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader.value()); - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", request); - return; - } - Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - String responseTopic = bytesToString(responseTopicHeader.value()); - try { - pendingRequestCount.getAndIncrement(); - Request decodedRequest = requestTemplate.decode(request); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), - response -> { - pendingRequestCount.decrementAndGet(); - reply(requestId, responseTopic, response); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timedout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - }); - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - - private void reply(UUID requestId, String topic, Response response) { - responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java new file mode 100644 index 0000000000..bb541fb626 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueAdmin { + + void createTopicIfNotExists(String topic); + + void destroy(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java new file mode 100644 index 0000000000..f21ad80d10 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueCallback { + + void onSuccess(TbQueueMsgMetadata metadata); + + void onFailure(Throwable t); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 8ec5cd1d97..1b2f984093 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -13,18 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; -import org.apache.kafka.clients.producer.Partitioner; -import org.apache.kafka.common.PartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; +import java.util.Set; -/** - * Created by ashvayka on 25.09.18. - */ -public interface TbKafkaPartitioner extends Partitioner { +public interface TbQueueConsumer { + + String getTopic(); + + void subscribe(); + + void subscribe(Set partitions); + + void unsubscribe(); + + List poll(long durationInMillis); - int partition(String topic, String key, T value, byte[] encodedValue, List partitions); + void commit(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java index 5e5aa0591a..609cbd2107 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; /** * Created by ashvayka on 05.10.18. */ -public interface TbKafkaHandler { +public interface TbQueueHandler { ListenableFuture handle(Request request); diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java index 6e433e71a9..b0e4990305 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; import java.util.UUID; -public interface TbKafkaRequestIdExtractor { +public interface TbQueueMsg { - UUID extractRequestId(T value); + UUID getKey(); + TbQueueMsgHeaders getHeaders(); + + byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java new file mode 100644 index 0000000000..4cb38f6685 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.thingsboard.server.queue.TbQueueMsg; + +public interface TbQueueMsgDecoder { + + T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java new file mode 100644 index 0000000000..f205ed676f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import java.util.Map; + +public interface TbQueueMsgHeaders { + + byte[] put(String key, byte[] value); + + byte[] get(String key); + + Map getData(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java new file mode 100644 index 0000000000..0c24fec438 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueMsgMetadata { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java new file mode 100644 index 0000000000..ef308291a9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +public interface TbQueueProducer { + + void init(); + + String getDefaultTopic(); + + void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java similarity index 68% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index ef6b565817..20530360d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue; -/** - * @author Andrew Shvayka - */ -public interface DiscoveryServiceListener { +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueRequestTemplate { + + void init(); - void onServerAdded(ServerInstance server); + ListenableFuture send(Request request); - void onServerUpdated(ServerInstance server); + void stop(); - void onServerRemoved(ServerInstance server); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java new file mode 100644 index 0000000000..c823b3df88 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueResponseTemplate { + + void init(TbQueueHandler handler); + + void stop(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java new file mode 100644 index 0000000000..336514b7b2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusAdmin implements TbQueueAdmin { + + private final Set queues = ConcurrentHashMap.newKeySet(); + + private final ManagementClient client; + + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + ConnectionStringBuilder builder = new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + "queues", + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + + client = new ManagementClient(builder); + + try { + client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to get queues.", e); + throw new RuntimeException("Failed to get queues.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + if (queues.contains(topic)) { + return; + } + + try { + client.createQueue(topic); + queues.add(topic); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to create queue: [{}]", topic, e); + } + } + + public void destroy() { + try { + client.close(); + } catch (IOException e) { + log.error("Failed to close ManagementClient."); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java new file mode 100644 index 0000000000..cca599d59a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.microsoft.azure.servicebus.TransactionContext; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; +import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; +import com.microsoft.azure.servicebus.primitives.MessagingEntityType; +import com.microsoft.azure.servicebus.primitives.MessagingFactory; +import com.microsoft.azure.servicebus.primitives.SettleModePair; +import lombok.extern.slf4j.Slf4j; +import org.apache.qpid.proton.amqp.messaging.Data; +import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; +import org.apache.qpid.proton.amqp.transport.SenderSettleMode; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbServiceBusConsumerTemplate implements TbQueueConsumer { + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbServiceBusSettings serviceBusSettings; + + private final Gson gson = new Gson(); + + private Set receivers; + private volatile Set partitions; + private volatile boolean subscribed; + private volatile boolean stopped = false; + private Map> pendingMessages = new ConcurrentHashMap<>(); + private volatile int messagesPerQueue; + + public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.serviceBusSettings = serviceBusSettings; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + receivers.forEach(CoreMessageReceiver::closeAsync); + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); + subscribed = true; + } + + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to parse message.", e); + throw new RuntimeException("Failed to parse message.", e); + } + }).collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + private void createReceivers() { + List> receiverFutures = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .map(queue -> { + MessagingFactory factory; + try { + factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to create factory for the queue [{}]", queue); + throw new RuntimeException("Failed to create the factory", e); + } + + return CoreMessageReceiver.create(factory, queue, queue, 0, + new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), + MessagingEntityType.QUEUE); + }).collect(Collectors.toList()); + + try { + receivers = new HashSet<>(fromList(receiverFutures).get()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to create receivers", e); + } + } + } + + private ConnectionStringBuilder createConnection(String queue) { + admin.createTopicIfNotExists(queue); + return new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + queue, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + } + + private CompletableFuture> fromList(List> futures) { + CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; + futures.toArray(arrayFuture); + + return CompletableFuture + .allOf(arrayFuture) + .thenApply(v -> futures + .stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + } + + @Override + public void commit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); + } + + private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java new file mode 100644 index 0000000000..5d9a931378 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java @@ -0,0 +1,110 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.Message; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.ReceiveMode; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +public class TbServiceBusProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbServiceBusSettings serviceBusSettings; + private final Map clients = new HashMap<>(); + private ExecutorService executorService; + + public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.serviceBusSettings = serviceBusSettings; + executorService = Executors.newSingleThreadExecutor(); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); + CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); + future.whenCompleteAsync((success, err) -> { + if (err != null) { + callback.onFailure(err); + } else { + callback.onSuccess(null); + } + }, executorService); + } + + @Override + public void stop() { + clients.forEach((t, client) -> { + try { + client.close(); + } catch (ServiceBusException e) { + log.error("Failed to close QueueClient.", e); + } + }); + + if (executorService != null) { + executorService.shutdownNow(); + } + } + + private QueueClient getClient(String topic) { + return clients.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + ConnectionStringBuilder builder = + new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + topic, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + try { + return new QueueClient(builder, ReceiveMode.PEEKLOCK); + } catch (InterruptedException | ServiceBusException e) { + log.error("Failed to create new client for the Queue: [{}]", topic, e); + throw new RuntimeException("Failed to create new client for the Queue", e); + } + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java new file mode 100644 index 0000000000..d872dcec0b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +@Component +@Data +public class TbServiceBusSettings { + @Value("${queue.service_bus.namespace_name}") + private String namespaceName; + @Value("${queue.service_bus.sas_key_name}") + private String sasKeyName; + @Value("${queue.service_bus.sas_key}") + private String sasKey; + @Value("${queue.service_bus.max_messages}") + private int maxMessages; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 0c68c8dd5d..ec3b0b537a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -13,19 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; - -import lombok.extern.slf4j.Slf4j; +package org.thingsboard.server.queue.common; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public abstract class AbstractTbKafkaTemplate { +public class AbstractTbQueueTemplate { + protected static final String REQUEST_ID_HEADER = "requestId"; + protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + protected static final String REQUEST_TIME = "requestTime"; + protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); @@ -47,4 +45,14 @@ public abstract class AbstractTbKafkaTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } + + protected static byte[] longToBytes(long x) { + ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); + longBuffer.putLong(0, x); + return longBuffer.array(); + } + + protected static long bytesToLong(byte[] bytes) { + return ByteBuffer.wrap(bytes).getLong(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java index a01411e48f..aa87df031a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index df41a7e95f..7584e8c2d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -13,23 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.common; -import io.grpc.stub.StreamObserver; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.queue.TbQueueMsg; import java.util.UUID; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionCreateRequestMsg { +public class DefaultTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final byte[] data; + private final DefaultTbQueueMsgHeaders headers; - private final UUID msgUid; - private final ServerAddress remoteAddress; - private final StreamObserver responseObserver; + public DefaultTbQueueMsg(TbQueueMsg msg) { + this.key = msg.getKey(); + this.data = msg.getData(); + DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + msg.getHeaders().getData().forEach(headers::put); + this.headers = headers; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java new file mode 100644 index 0000000000..36c3dd0999 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.HashMap; +import java.util.Map; + +public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { + + protected final Map data = new HashMap<>(); + + @Override + public byte[] put(String key, byte[] value) { + return data.put(key, value); + } + + @Override + public byte[] get(String key) { + return data.get(key); + } + + @Override + public Map getData() { + return data; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 2a7bca119f..f02ae63441 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -13,26 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -41,15 +37,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; -/** - * Created by ashvayka on 25.09.18. - */ @Slf4j -public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { +public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate + implements TbQueueRequestTemplate { - private final TBKafkaProducerTemplate requestTemplate; - private final TBKafkaConsumerTemplate responseTemplate; - private final ConcurrentMap> pendingRequests; + private final TbQueueAdmin queueAdmin; + private final TbQueueProducer requestTemplate; + private final TbQueueConsumer responseTemplate; + private final ConcurrentMap> pendingRequests; private final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; @@ -60,12 +55,14 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe private volatile boolean stopped = false; @Builder - public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, - TBKafkaConsumerTemplate responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { + public DefaultTbQueueRequestTemplate(TbQueueAdmin queueAdmin, + TbQueueProducer requestTemplate, + TbQueueConsumer responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { + this.queueAdmin = queueAdmin; this.requestTemplate = requestTemplate; this.responseTemplate = responseTemplate; this.pendingRequests = new ConcurrentHashMap<>(); @@ -81,20 +78,9 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe } } + @Override public void init() { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.requestTemplate.getSettings()); - CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) { - log.trace("[{}] Topic already exists. ", responseTemplate.getTopic()); - } else { - log.info("[{}] Failed to create topic: {}", responseTemplate.getTopic(), e.getMessage(), e); - throw new RuntimeException(e); - } - - } + queueAdmin.createTopicIfNotExists(responseTemplate.getTopic()); this.requestTemplate.init(); tickTs = System.currentTimeMillis(); responseTemplate.subscribe(); @@ -102,44 +88,29 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe long nextCleanupMs = 0L; while (!stopped) { try { - ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); - if (responses.count() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.count()); + List responses = responseTemplate.poll(pollInterval); + if (responses.size() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } else { + continue; } responses.forEach(response -> { - log.trace("Received response to Kafka Template request: {}", response); - Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - Response decodedResponse = null; - UUID requestId = null; + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId; if (requestIdHeader == null) { - try { - decodedResponse = responseTemplate.decode(response); - requestId = responseTemplate.extractRequestId(decodedResponse); - } catch (IOException e) { - log.error("Failed to decode response", e); - } - } else { - requestId = bytesToUuid(requestIdHeader.value()); - } - if (requestId == null) { log.error("[{}] Missing requestId in header and body", response); } else { - log.trace("[{}] Response received", requestId); + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received: {}", requestId, response); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); } else { - try { - if (decodedResponse == null) { - decodedResponse = responseTemplate.decode(response); - } - expectedResponse.future.set(decodedResponse); - } catch (IOException e) { - expectedResponse.future.setException(e); - } + expectedResponse.future.set(response); } } }); + responseTemplate.commit(); tickTs = System.currentTimeMillis(); tickSize = pendingRequests.size(); if (nextCleanupMs < tickTs) { @@ -155,10 +126,6 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe }); nextCleanupMs = tickTs + maxRequestTimeout; } - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } } catch (Throwable e) { log.warn("Failed to obtain responses from queue.", e); try { @@ -171,30 +138,46 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe }); } + @Override public void stop() { stopped = true; + + if (responseTemplate != null) { + responseTemplate.unsubscribe(); + } + + if (requestTemplate != null) { + requestTemplate.stop(); + } + if (internalExecutor) { executor.shutdownNow(); } } - public ListenableFuture post(String key, Request request) { + @Override + public ListenableFuture send(Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); - List
headers = new ArrayList<>(2); - headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); - headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); + request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); + request.getHeaders().put(REQUEST_TIME, longToBytes(System.currentTimeMillis())); SettableFuture future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); - requestTemplate.send(key, request, headers, (metadata, exception) -> { - if (exception != null) { - log.trace("[{}] Failed to post the request", requestId, exception); - } else { - log.trace("[{}] Posted the request: {}", requestId, metadata); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); + requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}] Request sent: {}", requestId, metadata); + } + + @Override + public void onFailure(Throwable t) { + pendingRequests.remove(requestId); + future.setException(t); } }); return future; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java new file mode 100644 index 0000000000..96891ee7d8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate + implements TbQueueResponseTemplate { + + private final TbQueueConsumer requestTemplate; + private final TbQueueProducer responseTemplate; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, + TbQueueProducer responseTemplate, + TbQueueHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + public void init(TbQueueHandler handler) { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + List requests = requestTemplate.poll(pollInterval); + + if (requests.isEmpty()) { + continue; + } + + requests.forEach(request -> { + long currentTime = System.currentTimeMillis(); + long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); + if (requestTime + requestTimeout >= currentTime) { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + } + }); + requestTemplate.commit(); + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (requestTemplate != null) { + requestTemplate.unsubscribe(); + } + if (responseTemplate != null) { + responseTemplate.stop(); + } + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java new file mode 100644 index 0000000000..f860d8dcc9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { + + private final AtomicInteger tbQueueCallbackCount; + private final TbMsgCallback tbMsgCallback; + + public MultipleTbQueueTbMsgCallbackWrapper(int tbQueueCallbackCount, TbMsgCallback tbMsgCallback) { + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); + this.tbMsgCallback = tbMsgCallback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (tbQueueCallbackCount.decrementAndGet() <= 0) { + tbMsgCallback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java new file mode 100644 index 0000000000..07417c4a9e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class TbProtoJsQueueMsg extends TbProtoQueueMsg { + + public TbProtoJsQueueMsg(UUID key, T value) { + super(key, value); + } + + public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + super(key, value, headers); + } + + @Override + public byte[] getData() { + try { + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java new file mode 100644 index 0000000000..2eb76950dc --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbProtoQueueMsg implements TbQueueMsg { + + private final UUID key; + protected final T value; + private final TbQueueMsgHeaders headers; + + public TbProtoQueueMsg(UUID key, T value) { + this(key, value, new DefaultTbQueueMsgHeaders()); + } + + public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + this.key = key; + this.value = value; + this.headers = headers; + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return value.toByteArray(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java new file mode 100644 index 0000000000..8bd9c3516b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { + + private final TbMsgCallback tbMsgCallback; + + public TbQueueTbMsgCallbackWrapper(TbMsgCallback tbMsgCallback) { + this.tbMsgCallback = tbMsgCallback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + tbMsgCallback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java similarity index 55% rename from common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index 5d47425e43..f0039d0f38 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -13,21 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.timeout; +package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.common.msg.MsgType; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -/** - * @author Andrew Shvayka - */ -public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { +import java.util.Set; - public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { - super(id, timeout); - } - @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; +public class ClusterTopologyChangeEvent extends ApplicationEvent { + + @Getter + private final Set serviceQueueKeys; + + public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { + super(source); + this.serviceQueueKeys = serviceQueueKeys; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java index 711677dc78..1b40545cde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -25,11 +24,10 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = - new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); - public void put(long hash, ServerInstance instance) { + public void put(long hash, T instance) { circle.put(hash, instance); } @@ -45,7 +43,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -53,11 +51,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public ServerInstance get(Long hash) { + public T get(Long hash) { return circle.get(hash); } public void log() { - circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); + circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java new file mode 100644 index 0000000000..df25ea3ba9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -0,0 +1,339 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class ConsistentHashPartitionService implements PartitionService { + + @Value("${queue.core.topic}") + private String coreTopic; + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.partitions.hash_function_name:murmur3_128}") + private String hashFunctionName; + @Value("${queue.partitions.virtual_nodes_size:16}") + private Integer virtualNodesSize; + + private final ApplicationEventPublisher applicationEventPublisher; + private final TbServiceInfoProvider serviceInfoProvider; + private final TenantRoutingInfoService tenantRoutingInfoService; + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); + + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); + + private Map tbCoreNotificationTopics = new HashMap<>(); + private Map tbRuleEngineNotificationTopics = new HashMap<>(); + private List currentOtherServices; + + private HashFunction hashFunction; + + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { + this.serviceInfoProvider = serviceInfoProvider; + this.tenantRoutingInfoService = tenantRoutingInfoService; + this.applicationEventPublisher = applicationEventPublisher; + } + + @PostConstruct + public void init() { + this.hashFunction = forName(hashFunctionName); + partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); + partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType), tenantId, entityId); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); + } + + private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { + int hash = hashFunction.newHasher() + .putLong(entityId.getId().getMostSignificantBits()) + .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); + Integer partitionSize = partitionSizes.get(serviceQueue); + int partition; + if (partitionSize != null) { + partition = Math.abs(hash % partitionSize); + } else { + //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. + partition = 0; + } + boolean isolatedTenant = isIsolated(serviceQueue, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); + } + + @Override + public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + logServiceInfo(currentService); + otherServices.forEach(this::logServiceInfo); + Map> circles = new HashMap<>(); + addNode(circles, currentService); + for (ServiceInfo other : otherServices) { + addNode(circles, other); + } + ConcurrentMap> oldPartitions = myPartitions; + TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); + myPartitions = new ConcurrentHashMap<>(); + partitionSizes.forEach((type, size) -> { + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); + for (int i = 0; i < size; i++) { + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); + if (currentService.equals(serviceInfo)) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); + } + } + }); + myPartitions.forEach((serviceQueueKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) + .collect(Collectors.toSet()); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); + } + }); + tpiCache.clear(); + + if (currentOtherServices == null) { + currentOtherServices = new ArrayList<>(otherServices); + } else { + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); + currentOtherServices = otherServices; + currentMap.forEach((key, list) -> { + if (!list.equals(newMap.get(key))) { + changes.add(key); + } + }); + currentMap.keySet().forEach(newMap::remove); + changes.addAll(newMap.keySet()); + if (!changes.isEmpty()) { + applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + } + } + } + + @Override + public Set getAllServiceIds(ServiceType serviceType) { + Set result = new HashSet<>(); + ServiceInfo current = serviceInfoProvider.getServiceInfo(); + if (current.getServiceTypesList().contains(serviceType.name())) { + result.add(current.getServiceId()); + } + for (ServiceInfo serviceInfo : currentOtherServices) { + if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { + result.add(serviceInfo.getServiceId()); + } + } + return result; + } + + @Override + public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { + switch (serviceType) { + case TB_CORE: + return tbCoreNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); + case TB_RULE_ENGINE: + return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); + default: + return buildNotificationsTopicPartitionInfo(serviceType, serviceId); + } + } + + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } + }); + return currentMap; + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { + return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceQueue)); + tpi.partition(partition); + ServiceQueueKey myPartitionsSearchKey; + if (isIsolated(serviceQueue, tenantId)) { + tpi.tenantId(tenantId); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); + } else { + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); + } + List partitions = myPartitions.get(myPartitionsSearchKey); + if (partitions != null) { + tpi.myPartition(partitions.contains(partition)); + } else { + tpi.myPartition(false); + } + return tpi.build(); + } + + private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return false; + } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + synchronized (tenantRoutingInfoMap) { + routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + tenantRoutingInfoMap.put(tenantId, routingInfo); + } + } + } + if (routingInfo == null) { + throw new RuntimeException("Tenant not found!"); + } + switch (serviceQueue.getType()) { + case TB_CORE: + return routingInfo.isIsolatedTbCore(); + case TB_RULE_ENGINE: + return routingInfo.isIsolatedTbRuleEngine(); + default: + return false; + } + } + + private void logServiceInfo(TransportProtos.ServiceInfo server) { + TenantId tenantId = getSystemOrIsolatedTenantId(server); + if (tenantId.isNullUid()) { + log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); + } else { + log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); + } + } + + private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { + return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); + } + + private void addNode(Map> circles, ServiceInfo instance) { + TenantId tenantId = getSystemOrIsolatedTenantId(instance); + for (String serviceTypeStr : instance.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } + } + + private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { + if (circle == null || circle.isEmpty()) { + return null; + } + Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + return circle.get(hash); + } + + private HashCode hash(ServiceInfo instance, int i) { + return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); + } + + public static HashFunction forName(String name) { + switch (name) { + case "murmur3_32": + return Hashing.murmur3_32(); + case "murmur3_128": + return Hashing.murmur3_128(); + case "crc32": + return Hashing.crc32(); + case "md5": + return Hashing.md5(); + default: + throw new IllegalArgumentException("Can't find hash function with name " + name); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java new file mode 100644 index 0000000000..1c60a03cdd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { + + @Getter + @Value("${service.id:#{null}}") + private String serviceId; + + @Getter + @Value("${service.type:monolith}") + private String serviceType; + + @Getter + @Value("${service.tenant_id:}") + private String tenantIdStr; + + @Autowired(required = false) + private TbQueueRuleEngineSettings ruleEngineSettings; + + private List serviceTypes; + private ServiceInfo serviceInfo; + private TenantId isolatedTenant; + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(serviceId)) { + try { + serviceId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current Service ID: {}", serviceId); + if (serviceType.equalsIgnoreCase("monolith")) { + serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); + } else { + serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); + } + ServiceInfo.Builder builder = ServiceInfo.newBuilder() + .setServiceId(serviceId) + .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + UUID tenantId; + if (!StringUtils.isEmpty(tenantIdStr)) { + tenantId = UUID.fromString(tenantIdStr); + isolatedTenant = new TenantId(tenantId); + } else { + tenantId = TenantId.NULL_UUID; + } + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + + if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { + for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { + TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() + .setName(queue.getName()) + .setTopic(queue.getTopic()) + .setPartitions(queue.getPartitions()).build(); + builder.addRuleEngineQueues(queueInfo); + } + } + + serviceInfo = builder.build(); + } + + @Override + public ServiceInfo getServiceInfo() { + return serviceInfo; + } + + @Override + public boolean isService(ServiceType serviceType) { + return serviceTypes.contains(serviceType); + } + + @Override + public Optional getIsolatedTenant() { + return Optional.ofNullable(isolatedTenant); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java new file mode 100644 index 0000000000..36510261d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +public interface DiscoveryService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 009a7801e7..9017823efb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -13,55 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.DependsOn; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; import java.util.Collections; -import java.util.List; -/** - * @author Andrew Shvayka - */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") public class DummyDiscoveryService implements DiscoveryService { - @Autowired - private ServerInstanceService serverInstance; - - @PostConstruct - public void init() { - log.info("Initializing..."); - } - - @Override - public void publishCurrentServer() { - //Do nothing - } + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; - @Override - public void unpublishCurrentServer() { - //Do nothing - } - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); + public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; } - @Override - public List getOtherServers() { - return Collections.emptyList(); + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java new file mode 100644 index 0000000000..19b8f665a8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + + +public class PartitionChangeEvent extends ApplicationEvent { + + @Getter + private final ServiceQueueKey serviceQueueKey; + @Getter + private final Set partitions; + + public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { + super(source); + this.serviceQueueKey = serviceQueueKey; + this.partitions = partitions; + } + + public ServiceType getServiceType() { + return serviceQueueKey.getServiceQueue().getType(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java new file mode 100644 index 0000000000..e3b3e76b80 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; +import java.util.Set; + +/** + * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} + */ +public interface PartitionService { + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + + TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); + + /** + * Received from the Discovery service when network topology is changed. + * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + */ + void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + + /** + * Get all active service ids by service type + * @param serviceType to filter the list of services + * @return list of all active services + */ + Set getAllServiceIds(ServiceType serviceType); + + /** + * Each Service should start a consumer for messages that target individual service instance based on serviceId. + * This topic is likely to have single partition, and is always assigned to the service. + * @param serviceType + * @param serviceId + * @return + */ + TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 0d401e4d9c..fcc7c7a60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -13,19 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import java.util.function.Consumer; +import java.util.Optional; -/** - * Created by ashvayka on 05.10.18. - */ -public interface RuleEngineTransportService { +public interface TbServiceInfoProvider { + + String getServiceId(); + + ServiceInfo getServiceInfo(); - void process(String nodeId, DeviceActorToTransportMsg msg); + boolean isService(ServiceType serviceType); - void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + Optional getIsolatedTenant(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java similarity index 71% rename from application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index 811713819a..07c4c2d9c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -13,18 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.discovery; -import akka.actor.ActorRef; import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ @Data -public final class SessionActorInfo { - protected final UUID sessionId; - protected final ActorRef actor; +public class TenantRoutingInfo { + private final TenantId tenantId; + private final boolean isolatedTbCore; + private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java new file mode 100644 index 0000000000..685a36df30 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantRoutingInfoService { + + TenantRoutingInfo getRoutingInfo(TenantId tenantId); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java new file mode 100644 index 0000000000..37661e85e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.AllArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceQueue; + +import java.util.Objects; + +@AllArgsConstructor +public class TopicPartitionInfoKey { + private ServiceQueue serviceQueue; + private TenantId isolatedTenantId; + private int partition; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; + return partition == that.partition && + serviceQueue.equals(that.serviceQueue) && + Objects.equals(isolatedTenantId, that.isolatedTenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceQueue, isolatedTenantId, partition); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index e1cdd5f83e..e761a229c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue.discovery; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.SerializationException; -import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -32,22 +30,14 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.utils.MiscUtils; +import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -59,9 +49,6 @@ import java.util.stream.Collectors; import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; -/** - * @author Andrew Shvayka - */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j @@ -78,42 +65,29 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @Value("${zk.zk_dir}") private String zkDir; - private String zkNodesDir; - - @Autowired - private ServerInstanceService serverInstance; - - @Autowired - @Lazy - private TelemetrySubscriptionService tsSubService; - - @Autowired - @Lazy - private DeviceStateService deviceStateService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - @Lazy - private ClusterRoutingService routingService; + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private ExecutorService reconnectExecutorService; - private CuratorFramework client; private PathChildrenCache cache; private String nodePath; + private String zkNodesDir; private volatile boolean stopped = true; + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; + } + @PostConstruct public void init() { log.info("Initializing..."); - Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); + Assert.hasLength(zkUrl, missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); @@ -123,53 +97,47 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi initZkClient(); } - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) {} - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); + private List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return TransportProtos.ServiceInfo.parseFrom(cd.getData()); + } catch (NoSuchElementException | InvalidProtocolBufferException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); } - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } else { + log.info("Received application ready event. Starting current ZK node."); + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); } - @Override public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); } else { try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + log.info("[{}] Creating ZK node for current instance", self.getServiceId()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); + log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -183,10 +151,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + TransportProtos.ServiceInfo registeredServerInfo = null; + registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); + if (self.equals(registeredServerInfo)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -197,9 +165,9 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } - private ConnectionStateListener checkReconnect(ServerInstance self) { + private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -223,8 +191,25 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - @Override - public void unpublishCurrentServer() { + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } + } + + private void unpublishCurrentServer() { try { if (nodePath != null) { client.delete().forPath(nodePath); @@ -235,41 +220,26 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) { + } + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); } - @Override - public List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); - } catch (NoSuchElementException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting current ZK node."); - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); + public static String missingProperty(String propertyName) { + return "The " + propertyName + " property need to be set!"; } @Override @@ -297,34 +267,23 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); return; } - ServerInstance instance; + TransportProtos.ServiceInfo instance; try { - ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); - instance = new ServerInstance(serverAddress); - } catch (SerializationException e) { + instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); + } catch (InvalidProtocolBufferException e) { log.error("Failed to decode server instance for node {}", data.getPath(), e); throw e; } - log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); + log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); switch (pathChildrenCacheEvent.getType()) { case CHILD_ADDED: - routingService.onServerAdded(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerAdded(instance); - break; case CHILD_UPDATED: - routingService.onServerUpdated(instance); - actorService.onServerUpdated(instance); - break; case CHILD_REMOVED: - routingService.onServerRemoved(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerRemoved(instance); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); break; default: break; } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java index a6a881ecde..267b34b204 100644 --- a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.environment; +package org.thingsboard.server.queue.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("Thingsboard server environment: ", log); + Environment.logEnv("ThingsBoard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java new file mode 100644 index 0000000000..98470eafba --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.util.UUID; + +public class KafkaTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final TbQueueMsgHeaders headers; + private final byte[] data; + + public KafkaTbQueueMsg(ConsumerRecord record) { + this.key = UUID.fromString(record.key()); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + record.headers().forEach(header -> { + headers.put(header.key(), header.value()); + }); + this.headers = headers; + this.data = record.value(); + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return data; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java index bb87d2ea88..5a9eaa632c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java @@ -13,17 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.kafka; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.thingsboard.server.queue.TbQueueMsgMetadata; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionClosedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; +@AllArgsConstructor +public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { + private RecordMetadata metadata; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java new file mode 100644 index 0000000000..b92b094af1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.common.errors.TopicExistsException; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaAdmin implements TbQueueAdmin { + + private final AdminClient client; + private final Map topicConfigs; + private final Set topics = ConcurrentHashMap.newKeySet(); + + private final short replicationFactor; + + public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { + client = AdminClient.create(settings.toProps()); + this.topicConfigs = topicConfigs; + + try { + topics.addAll(client.listTopics().names().get()); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to get all topics.", e); + } + + replicationFactor = settings.getReplicationFactor(); + } + + @Override + public void createTopicIfNotExists(String topic) { + if (topics.contains(topic)) { + return; + } + try { + NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + createTopic(newTopic).values().get(topic).get(); + topics.add(topic); + } catch (ExecutionException ee) { + if (ee.getCause() instanceof TopicExistsException) { + //do nothing + } else { + log.warn("[{}] Failed to create topic", topic, ee); + throw new RuntimeException(ee); + } + } catch (Exception e) { + log.warn("[{}] Failed to create topic", topic, e); + throw new RuntimeException(e); + } + + } + + @Override + public void destroy() { + if (client != null) { + client.close(); + } + } + + public CreateTopicsResult createTopic(NewTopic topic) { + return client.createTopics(Collections.singletonList(topic)); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java new file mode 100644 index 0000000000..49d1d74102 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -0,0 +1,159 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaConsumerTemplate implements TbQueueConsumer { + + private final TbQueueAdmin admin; + private final KafkaConsumer consumer; + private final TbKafkaDecoder decoder; + private volatile boolean subscribed; + private volatile Set partitions; + private final Lock consumerLock; + + @Getter + private final String topic; + + @Builder + private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + String clientId, String groupId, String topic, + boolean autoCommit, int autoCommitIntervalMs, + int maxPollRecords, + TbQueueAdmin admin) { + Properties props = settings.toProps(); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + if (groupId != null) { + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + } + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + if (maxPollRecords > 0) { + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + } + this.admin = admin; + this.consumer = new KafkaConsumer<>(props); + this.decoder = decoder; + this.topic = topic; + this.consumerLock = new ReentrantLock(); + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + try { + consumerLock.lock(); + + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } + + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + } finally { + consumerLock.unlock(); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + consumerLock.lock(); + consumer.commitAsync(); + } finally { + consumerLock.unlock(); + } + } + + @Override + public void unsubscribe() { + try { + consumerLock.lock(); + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); + } + } finally { + consumerLock.unlock(); + } + } + + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(new KafkaTbQueueMsg(record)); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java index ab196d5863..6e3ea1e4a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; + +import org.thingsboard.server.queue.TbQueueMsg; import java.io.IOException; @@ -22,6 +24,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(byte[] data) throws IOException; + T decode(TbQueueMsg msg) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java index 22f65c58a0..b3c4dec8ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java new file mode 100644 index 0000000000..4f26f51da7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; + +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaProducerTemplate implements TbQueueProducer { + + private final KafkaProducer producer; + + @Getter + private final String defaultTopic; + + @Getter + private final TbKafkaSettings settings; + + private final TbQueueAdmin admin; + + private final Set topics; + + @Builder + private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { + Properties props = settings.toProps(); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + if (!StringUtils.isEmpty(clientId)) { + props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + } + this.settings = settings; + this.producer = new KafkaProducer<>(props); + this.defaultTopic = defaultTopic; + this.admin = admin; + topics = ConcurrentHashMap.newKeySet(); + } + + @Override + public void init() { + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + createTopicIfNotExist(tpi); + String key = msg.getKey().toString(); + byte[] data = msg.getData(); + ProducerRecord record; + Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); + record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + producer.send(record, (metadata, exception) -> { + if (exception == null) { + if (callback != null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } + } else { + if (callback != null) { + callback.onFailure(exception); + } else { + log.warn("Producer template failure: {}", exception.getMessage(), exception); + } + } + }); + } + + private void createTopicIfNotExist(TopicPartitionInfo tpi) { + if (topics.contains(tpi)) { + return; + } + admin.createTopicIfNotExists(tpi.getFullTopicName()); + topics.add(tpi); + } + + @Override + public void stop() { + if (producer != null) { + producer.close(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java new file mode 100644 index 0000000000..b6de7db50d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Data; + +/** + * Created by ashvayka on 25.09.18. + */ +@Data +public class TbKafkaProperty { + + private String key; + private String value; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 6446643a18..66121cb215 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import java.util.List; @@ -28,31 +29,32 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") @Component public class TbKafkaSettings { - static final String REQUEST_ID_HEADER = "requestId"; - static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - - @Value("${kafka.bootstrap.servers}") + @Value("${queue.kafka.bootstrap.servers}") private String servers; - @Value("${kafka.acks}") + @Value("${queue.kafka.acks}") private String acks; - @Value("${kafka.retries}") + @Value("${queue.kafka.retries}") private int retries; - @Value("${kafka.batch.size}") + @Value("${queue.kafka.batch.size}") private int batchSize; - @Value("${kafka.linger.ms}") + @Value("${queue.kafka.linger.ms}") private long lingerMs; - @Value("${kafka.buffer.memory}") + @Value("${queue.kafka.buffer.memory}") private long bufferMemory; + @Value("${queue.kafka.replication_factor}") + @Getter + private short replicationFactor; + @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java new file mode 100644 index 0000000000..4b68abf929 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class TbKafkaTopicConfigs { + @Value("${queue.kafka.topic-properties.core}") + private String coreProperties; + @Value("${queue.kafka.topic-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.kafka.topic-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.kafka.topic-properties.notifications}") + private String notificationsProperties; + @Value("${queue.kafka.topic-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java new file mode 100644 index 0000000000..f51f5b8ae0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +@Slf4j +public final class InMemoryStorage { + private static InMemoryStorage instance; + private final ConcurrentHashMap> storage; + + private InMemoryStorage() { + storage = new ConcurrentHashMap<>(); + } + + public static InMemoryStorage getInstance() { + if (instance == null) { + synchronized (InMemoryStorage.class) { + if (instance == null) { + instance = new InMemoryStorage(); + } + } + } + return instance; + } + + public boolean put(String topic, TbQueueMsg msg) { + return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); + } + + public List get(String topic) throws InterruptedException { + if (storage.containsKey(topic)) { + List entities; + T first = (T) storage.get(topic).poll(); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 999); + for (TbQueueMsg other : otherList) { + entities.add((T) other); + } + } else { + entities = Collections.emptyList(); + } + return entities; + } + return Collections.emptyList(); + } + + /** + * Used primarily for testing. + */ + public void cleanup() { + storage.clear(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java new file mode 100644 index 0000000000..a619eda132 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class InMemoryTbQueueConsumer implements TbQueueConsumer { + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + private volatile Set partitions; + private volatile boolean stopped; + private volatile boolean subscribed; + + public InMemoryTbQueueConsumer(String topic) { + this.topic = topic; + stopped = false; + } + + private final String topic; + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = true; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = true; + } + + @Override + public void unsubscribe() { + stopped = true; + } + + @Override + public List poll(long durationInMillis) { + if (subscribed) { + List messages = partitions + .stream() + .map(tpi -> { + try { + return storage.get(tpi.getFullTopicName()); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Queue was interrupted.", e); + } + return Collections.emptyList(); + } + }) + .flatMap(List::stream) + .map(msg -> (T) msg).collect(Collectors.toList()); + if (messages.size() > 0) { + return messages; + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to sleep.", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java new file mode 100644 index 0000000000..cfcd788a16 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.Data; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +@Data +public class InMemoryTbQueueProducer implements TbQueueProducer { + + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + private final String defaultTopic; + + public InMemoryTbQueueProducer(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + @Override + public void init() { + + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + boolean result = storage.put(tpi.getFullTopicName(), msg); + if (result) { + if (callback != null) { + callback.onSuccess(null); + } + } else { + if (callback != null) { + callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); + } + } + } + + @Override + public void stop() { + + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java new file mode 100644 index 0000000000..76ff04c238 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java new file mode 100644 index 0000000000..a7349091bd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") +public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..3056eced2c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") +public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbAwsSqsSettings sqsSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.sqsSettings = sqsSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java new file mode 100644 index 0000000000..351cd4058c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -0,0 +1,124 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = + new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + + TbAwsSqsConsumerTemplate> consumerTemplate = + new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, + transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java new file mode 100644 index 0000000000..fbdbeaab0c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java new file mode 100644 index 0000000000..2855e577ee --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + InMemoryTbQueueProducer> producerTemplate = + new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumerTemplate = + new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + + templateBuilder.queueAdmin(new TbQueueAdmin() { + @Override + public void createTopicIfNotExists(String topic) {} + + @Override + public void destroy() {} + }); + + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java new file mode 100644 index 0000000000..72c6f7b8b9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -0,0 +1,250 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); + requestBuilder.admin(transportApiAdmin); + return requestBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java new file mode 100644 index 0000000000..ca10830981 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -0,0 +1,221 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") +public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..bcac399a64 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -0,0 +1,193 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java new file mode 100644 index 0000000000..2f7aab4779 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -0,0 +1,138 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.admin(transportApiAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(transportApiAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(requestBuilder.build()); + templateBuilder.responseTemplate(responseBuilder.build()); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(notificationAdmin); + return responseBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java new file mode 100644 index 0000000000..af0b276c1a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java new file mode 100644 index 0000000000..0a23d58e46 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") +public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..79501130ed --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -0,0 +1,138 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") +public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java new file mode 100644 index 0000000000..7e859e02ab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class PubSubTransportQueueFactory implements TbTransportQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java new file mode 100644 index 0000000000..72f45e5276 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") +public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbRabbitMqQueueArguments queueArguments) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java new file mode 100644 index 0000000000..d84bbedbd2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") +public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqQueueArguments queueArguments) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..a18f427fab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") +public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbRabbitMqSettings rabbitMqSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqSettings rabbitMqSettings, + TbRabbitMqQueueArguments queueArguments) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.rabbitMqSettings = rabbitMqSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java new file mode 100644 index 0000000000..841e004b3f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbRabbitMqQueueArguments queueArguments) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbRabbitMqProducerTemplate> producerTemplate = + new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + + TbRabbitMqConsumerTemplate> consumerTemplate = + new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java new file mode 100644 index 0000000000..8b01c38607 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java new file mode 100644 index 0000000000..523ac88bd1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -0,0 +1,124 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") +public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..8a2f9b396b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -0,0 +1,107 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") +public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java new file mode 100644 index 0000000000..33c5277ca5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = + new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + + TbQueueConsumer> consumerTemplate = + new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java new file mode 100644 index 0000000000..59a8f449b0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbCoreQueueFactory { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> createTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreMsgProducer(); + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToCoreMsgConsumer(); + + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToCoreNotificationsMsgConsumer(); + + /** + * Used to consume Transport API Calls + * + * @return + */ + TbQueueConsumer> createTransportApiRequestConsumer(); + + /** + * Used to push replies to Transport API Calls + * + * @return + */ + TbQueueProducer> createTransportApiResponseProducer(); + + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java new file mode 100644 index 0000000000..9fe5de0bcc --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { + + private final TbCoreQueueFactory tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java new file mode 100644 index 0000000000..34c6ee577f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for providing various Producers to other services. + */ +public interface TbQueueProducerProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java new file mode 100644 index 0000000000..69de7ca3e2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { + + private final TbRuleEngineQueueFactory tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + + public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..561b3e84ea --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbRuleEngineQueueFactory { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> createTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreMsgProducer(); + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + * @param configuration + */ + //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary + TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); + + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); + + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java new file mode 100644 index 0000000000..dc1d2c449c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +public interface TbTransportQueueFactory { + + TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); + + TbQueueProducer> createRuleEngineMsgProducer(); + + TbQueueProducer> createTbCoreMsgProducer(); + + TbQueueConsumer> createTransportNotificationsConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java new file mode 100644 index 0000000000..bced3f83ac --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { + + private final TbTransportQueueFactory tbQueueProvider; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + + public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java new file mode 100644 index 0000000000..1241370d05 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -0,0 +1,186 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.Duration; +import com.google.pubsub.v1.ListSubscriptionsRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ProjectName; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.Topic; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +public class TbPubSubAdmin implements TbQueueAdmin { + private static final String ACK_DEADLINE = "ackDeadlineInSec"; + private static final String MESSAGE_RETENTION = "messageRetentionInSec"; + + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + + private final TbPubSubSettings pubSubSettings; + private final Set topicSet = ConcurrentHashMap.newKeySet(); + private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + private final Map subscriptionProperties; + + public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.subscriptionProperties = subscriptionSettings; + + TopicAdminSettings topicAdminSettings; + try { + topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create TopicAdminSettings"); + throw new RuntimeException("Failed to create TopicAdminSettings."); + } + + SubscriptionAdminSettings subscriptionAdminSettings; + try { + subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create SubscriptionAdminSettings"); + throw new RuntimeException("Failed to create SubscriptionAdminSettings."); + } + + try { + topicAdminClient = TopicAdminClient.create(topicAdminSettings); + + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + topicSet.add(topic.getName()); + } + } catch (IOException e) { + log.error("Failed to get topics.", e); + throw new RuntimeException("Failed to get topics.", e); + } + + try { + subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + subscriptionSet.add(subscription.getName()); + } + } catch (IOException e) { + log.error("Failed to get subscriptions.", e); + throw new RuntimeException("Failed to get subscriptions.", e); + } + } + + @Override + public void createTopicIfNotExists(String partition) { + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + + if (topicSet.contains(topicName.toString())) { + createSubscriptionIfNotExists(partition, topicName); + return; + } + + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; + } + } + + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); + } + + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); + + if (subscriptionSet.contains(subscriptionName.toString())) { + return; + } + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; + } + } + + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); + + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); + + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } + + private void setAckDeadline(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(ACK_DEADLINE)) { + builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); + } + } + + private void setMessageRetention(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { + Duration duration = Duration + .newBuilder() + .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) + .build(); + builder.setMessageRetentionDuration(duration); + } + } + + @Override + public void destroy() { + if (topicAdminClient != null) { + topicAdminClient.close(); + } + if (subscriptionAdminClient != null) { + subscriptionAdminClient.close(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java new file mode 100644 index 0000000000..5dc795739a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -0,0 +1,219 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +public class TbPubSubConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbPubSubSettings pubSubSettings; + + private volatile boolean subscribed; + private volatile Set partitions; + private volatile Set subscriptionNames; + private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); + + private ExecutorService consumerExecutor; + private final SubscriberStub subscriber; + private volatile boolean stopped; + + private volatile int messagesPerTopic; + + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.pubSubSettings = pubSubSettings; + this.topic = topic; + this.decoder = decoder; + + try { + SubscriberStubSettings subscriberStubSettings = + SubscriberStubSettings.newBuilder() + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setTransportChannelProvider( + SubscriberStubSettings.defaultGrpcTransportProviderBuilder() + .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) + .build()) + .build(); + + this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); + } catch (IOException e) { + log.error("Failed to create subscriber.", e); + throw new RuntimeException("Failed to create subscriber.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + + if (subscriber != null) { + subscriber.close(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); + subscriptionNames.forEach(admin::createTopicIfNotExists); + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); + subscribed = true; + } + List messages; + try { + messages = receiveMessages(); + if (!messages.isEmpty()) { + List result = new ArrayList<>(); + messages.forEach(msg -> { + try { + result.add(decode(msg.getMessage())); + } catch (InvalidProtocolBufferException e) { + log.error("Failed decode record: [{}]", msg); + } + }); + return result; + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); + } + + private List receiveMessages() throws ExecutionException, InterruptedException { + List>> result = subscriptionNames.stream().map(subscriptionId -> { + String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); + PullRequest pullRequest = + PullRequest.newBuilder() + .setMaxMessages(messagesPerTopic) + .setReturnImmediately(false) // return immediately if messages are not available + .setSubscription(subscriptionName) + .build(); + + ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); + + return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { + if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { + List ackIds = new ArrayList<>(); + for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { + ackIds.add(message.getAckId()); + } + AcknowledgeRequest acknowledgeRequest = + AcknowledgeRequest.newBuilder() + .setSubscription(subscriptionName) + .addAllAckIds(ackIds) + .build(); + + acknowledgeRequests.add(acknowledgeRequest); + return pullResponse.getReceivedMessagesList(); + } + return null; + }, consumerExecutor); + + }).collect(Collectors.toList()); + + ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { + if (!CollectionUtils.isEmpty(listMessages)) { + return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + + return transform.get(); + } + + public T decode(PubsubMessage message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java new file mode 100644 index 0000000000..2cd2e1054e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.gson.Gson; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class TbPubSubProducerTemplate implements TbQueueProducer { + + private final Gson gson = new Gson(); + + private final String defaultTopic; + private final TbQueueAdmin admin; + private final TbPubSubSettings pubSubSettings; + + private final Map publisherMap = new ConcurrentHashMap<>(); + + private ExecutorService pubExecutor = Executors.newCachedThreadPool(); + + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { + this.defaultTopic = defaultTopic; + this.admin = admin; + this.pubSubSettings = pubSubSettings; + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); + pubsubMessageBuilder.setData(getMsg(msg)); + + Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); + ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); + + ApiFutures.addCallback(future, new ApiFutureCallback() { + public void onSuccess(String messageId) { + if (callback != null) { + callback.onSuccess(null); + } + } + + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, pubExecutor); + } + + @Override + public void stop() { + publisherMap.forEach((k, v) -> { + if (v != null) { + try { + v.shutdown(); + v.awaitTermination(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to shutdown PubSub client during destroy()", e); + } + } + }); + + if (pubExecutor != null) { + pubExecutor.shutdownNow(); + } + } + + private ByteString getMsg(T msg) { + String json = gson.toJson(new DefaultTbQueueMsg(msg)); + return ByteString.copyFrom(json.getBytes()); + } + + private Publisher getOrCreatePublisher(String topic) { + if (publisherMap.containsKey(topic)) { + return publisherMap.get(topic); + } else { + try { + admin.createTopicIfNotExists(topic); + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); + Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + publisherMap.put(topic, publisher); + return publisher; + } catch (IOException e) { + log.error("Failed to create topic [{}].", topic, e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java new file mode 100644 index 0000000000..d6c6d0e271 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.ByteArrayInputStream; +import java.io.IOException; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +@Component +@Data +public class TbPubSubSettings { + + @Value("${queue.pubsub.project_id}") + private String projectId; + + @Value("${queue.pubsub.service_account}") + private String serviceAccount; + + @Value("${queue.pubsub.max_msg_size}") + private int maxMsgSize; + + @Value("${queue.pubsub.max_messages}") + private int maxMessages; + + private CredentialsProvider credentialsProvider; + + @PostConstruct + private void init() throws IOException { + ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( + new ByteArrayInputStream(serviceAccount.getBytes())); + credentialsProvider = FixedCredentialsProvider.create(credentials); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java new file mode 100644 index 0000000000..e98f51ec7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +public class TbPubSubSubscriptionSettings { + @Value("${queue.pubsub.queue-properties.core}") + private String coreProperties; + @Value("${queue.pubsub.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.pubsub.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.pubsub.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.pubsub.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreSettings; + @Getter + private Map ruleEngineSettings; + @Getter + private Map transportApiSettings; + @Getter + private Map notificationsSettings; + @Getter + private Map jsExecutorSettings; + + @PostConstruct + private void init() { + coreSettings = getSettings(coreProperties); + ruleEngineSettings = getSettings(ruleEngineProperties); + transportApiSettings = getSettings(transportApiProperties); + notificationsSettings = getSettings(notificationsProperties); + jsExecutorSettings = getSettings(jsExecutorProperties); + } + + private Map getSettings(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java new file mode 100644 index 0000000000..3bef6deb84 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqAdmin implements TbQueueAdmin { + + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + private final Map arguments; + + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { + this.rabbitMqSettings = rabbitMqSettings; + this.arguments = arguments; + + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + try { + channel.queueDeclare(topic, false, false, false, arguments); + } catch (IOException e) { + log.error("Failed to bind queue: [{}]", topic, e); + } + } + + @Override + public void destroy() { + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close Chanel.", e); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close Connection.", e); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java new file mode 100644 index 0000000000..25d7719163 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.GetResponse; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@Slf4j +public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + private volatile Set partitions; + private volatile boolean subscribed; + private volatile Set queues; + private volatile boolean stopped; + + public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.rabbitMqSettings = rabbitMqSettings; + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + + queues.forEach(admin::createTopicIfNotExists); + subscribed = true; + } + + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode message: [{}].", message); + throw new RuntimeException("Failed to decode message.", e); + } + }).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } + } + + public T decode(GetResponse message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java new file mode 100644 index 0000000000..91b46213a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbRabbitMqSettings rabbitMqSettings; + private ListeningExecutorService producerExecutor; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.rabbitMqSettings = rabbitMqSettings; + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + AMQP.BasicProperties properties = new AMQP.BasicProperties(); + try { + channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); + if (callback != null) { + callback.onSuccess(null); + } + } catch (IOException e) { + log.error("Failed publish message: [{}].", msg, e); + if (callback != null) { + callback.onFailure(e); + } + } + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java new file mode 100644 index 0000000000..eb73e7dce6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqQueueArguments { + @Value("${queue.rabbitmq.queue-properties.core}") + private String coreProperties; + @Value("${queue.rabbitmq.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.rabbitmq.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.rabbitmq.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.rabbitmq.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreArgs; + @Getter + private Map ruleEngineArgs; + @Getter + private Map transportApiArgs; + @Getter + private Map notificationsArgs; + @Getter + private Map jsExecutorArgs; + + @PostConstruct + private void init() { + coreArgs = getArgs(coreProperties); + ruleEngineArgs = getArgs(ruleEngineProperties); + transportApiArgs = getArgs(transportApiProperties); + notificationsArgs = getArgs(notificationsProperties); + jsExecutorArgs = getArgs(jsExecutorProperties); + } + + private Map getArgs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } + return configs; + } + + private Object getObjectValue(String str) { + if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { + return Boolean.valueOf(str); + } else if (isNumeric(str)) { + return getNumericValue(str); + } + return str; + } + + private Object getNumericValue(String str) { + if (str.contains(".")) { + return Double.valueOf(str); + } else { + return Long.valueOf(str); + } + } + + private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); + + public boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + return PATTERN.matcher(strNum).matches(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java new file mode 100644 index 0000000000..e0156e6dc8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.ConnectionFactory; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +@Component +@Data +public class TbRabbitMqSettings { + @Value("${queue.rabbitmq.exchange_name:}") + private String exchangeName; + @Value("${queue.rabbitmq.host:}") + private String host; + @Value("${queue.rabbitmq.port:}") + private int port; + @Value("${queue.rabbitmq.virtual_host:}") + private String virtualHost; + @Value("${queue.rabbitmq.username:}") + private String username; + @Value("${queue.rabbitmq.password:}") + private String password; + @Value("${queue.rabbitmq.automatic_recovery_enabled:}") + private boolean automaticRecoveryEnabled; + @Value("${queue.rabbitmq.connection_timeout:}") + private int connectionTimeout; + @Value("${queue.rabbitmq.handshake_timeout:}") + private int handshakeTimeout; + + private ConnectionFactory connectionFactory; + + @PostConstruct + private void init() { + connectionFactory = new ConnectionFactory(); + connectionFactory.setHost(host); + connectionFactory.setPort(port); + connectionFactory.setVirtualHost(virtualHost); + connectionFactory.setUsername(username); + connectionFactory.setPassword(password); + connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); + connectionFactory.setConnectionTimeout(connectionTimeout); + connectionFactory.setHandshakeTimeout(handshakeTimeout); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index cd52948da8..1c260c754d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -13,20 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.settings; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -/** - * Created by ashvayka on 25.09.18. - */ @Data -public class TbKafkaProperty { +@Component +public class TbQueueCoreSettings { + + @Value("${queue.core.topic}") + private String topic; - private String key; - private String value; + @Value("${queue.core.partitions}") + private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java new file mode 100644 index 0000000000..cd492e87f1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRemoteJsInvokeSettings { + @Value("${queue.js.request_topic}") + private String requestTopic; + + @Value("${queue.js.response_topic_prefix}") + private String responseTopic; + + @Value("${queue.js.max_pending_requests}") + private long maxPendingRequests; + + @Value("${queue.js.response_poll_interval}") + private int responsePollInterval; + + @Value("${queue.js.response_auto_commit_interval}") + private int autoCommitInterval; + + @Value("${queue.js.max_requests_timeout}") + private long maxRequestsTimeout; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..9dfe715ece --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Data +@EnableAutoConfiguration +@Configuration +@ConfigurationProperties(prefix = "queue.rule-engine") +public class TbQueueRuleEngineSettings { + + private String topic; + private List queues; + + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. + // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + @PostConstruct + public void validate() { + queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { + log.error("Main queue is not configured in thingsboard.yml"); + return new RuntimeException("No \"Main\" queue configured!"); + }); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java new file mode 100644 index 0000000000..5ee58ab17e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportApiSettings { + + @Value("${queue.transport_api.requests_topic}") + private String requestsTopic; + + @Value("${queue.transport_api.responses_topic}") + private String responsesTopic; + + @Value("${queue.transport_api.max_pending_requests}") + private int maxPendingRequests; + + @Value("${queue.transport_api.max_requests_timeout}") + private int maxRequestsTimeout; + + @Value("${queue.transport_api.max_callback_threads}") + private int maxCallbackThreads; + + @Value("${queue.transport_api.request_poll_interval}") + private long requestPollInterval; + + @Value("${queue.transport_api.response_poll_interval}") + private long responsePollInterval; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index f4b14144fc..50da4f4a1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.executors; +package org.thingsboard.server.queue.settings; +import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.thingsboard.common.util.AbstractListeningExecutor; +@Data @Component -public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { +public class TbQueueTransportNotificationSettings { - @Value("${actors.cluster.grpc_callback_thread_pool_size}") - private int grpcCallbackExecutorThreadPoolSize; + @Value("${queue.transport.notifications_topic}") + private String notificationsTopic; - @Override - protected int getThreadPollSize() { - return grpcCallbackExecutorThreadPoolSize; - } + @Value("${queue.transport.poll_interval}") + private long transportPollInterval; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java index f0ff8ce9e1..0d21c59c9c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -13,15 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcBroadcastMsg { - private final ClusterAPIProtos.ClusterMessage msg; +public class TbRuleEngineQueueAckStrategyConfiguration { + + private String type; + private int retries; + private double failurePercentage; + private long pauseBetweenRetries; + } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java similarity index 62% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index 489a5bfa17..019a76953b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionConnectedMsg { +public class TbRuleEngineQueueConfiguration { + + private String name; + private String topic; + private int pollInterval; + private int partitions; + private long packProcessingTimeout; + private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; + private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; - private final ServerAddress remoteAddress; - private final UUID id; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java similarity index 77% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java index ed1bbf18b5..e39dc0c322 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport; +package org.thingsboard.server.queue.settings; -import org.thingsboard.server.common.data.Device; +import lombok.Data; -public interface SessionMsgProcessor { +@Data +public class TbRuleEngineQueueSubmitStrategyConfiguration { - void onDeviceAdded(Device device); + private String type; + private int batchSize; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java index 14e7504637..eb4eaed7ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.sqs; +import com.amazonaws.http.SdkHttpMetadata; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.TbQueueMsgMetadata; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionDisconnectedMsg { +@AllArgsConstructor +public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { - private final boolean client; - private final ServerAddress remoteAddress; + private final SdkHttpMetadata metadata; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java new file mode 100644 index 0000000000..8e293e6dff --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.CreateQueueRequest; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class TbAwsSqsAdmin implements TbQueueAdmin { + + private final Map attributes; + private final AmazonSQS sqsClient; + private final Set queues; + + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { + this.attributes = attributes; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .withRegion(sqsSettings.getRegion()) + .build(); + + queues = sqsClient + .listQueues() + .getQueueUrls() + .stream() + .map(this::getQueueNameFromUrl) + .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); + } + + @Override + public void createTopicIfNotExists(String topic) { + String queueName = topic.replaceAll("\\.", "_") + ".fifo"; + if (queues.contains(queueName)) { + return; + } + final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); + String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); + queues.add(getQueueNameFromUrl(queueUrl)); + } + + private String getQueueNameFromUrl(String queueUrl) { + int delimiterIndex = queueUrl.lastIndexOf("/"); + return queueUrl.substring(delimiterIndex + 1); + } + + @Override + public void destroy() { + if (sqsClient != null) { + sqsClient.shutdown(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java new file mode 100644 index 0000000000..3e71388844 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -0,0 +1,231 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { + + private static final int MAX_NUM_MSGS = 10; + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final AmazonSQS sqsClient; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbAwsSqsSettings sqsSettings; + + private final List pendingMessages = new CopyOnWriteArrayList<>(); + private volatile Set queueUrls; + private volatile Set partitions; + private ListeningExecutorService consumerExecutor; + private volatile boolean subscribed; + private volatile boolean stopped = false; + + public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + + if (sqsClient != null) { + sqsClient.shutdown(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); + subscribed = true; + } + + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .map(msg -> { + try { + return decode(msg); + } catch (IOException e) { + log.error("Failed to decode message: [{}]", msg); + return null; + } + }).filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", topic); + } else { + log.error("Failed to pool messages.", e); + } + } + } + return Collections.emptyList(); + } + + private ListenableFuture> poll(String url, int waitTimeSeconds) { + List>> result = new ArrayList<>(); + + for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { + result.add(consumerExecutor.submit(() -> { + ReceiveMessageRequest request = new ReceiveMessageRequest(); + request + .withWaitTimeSeconds(waitTimeSeconds) + .withMessageAttributeNames("headers") + .withQueueUrl(url) + .withMaxNumberOfMessages(MAX_NUM_MSGS); + return sqsClient.receiveMessage(request).getMessages(); + })); + } + return Futures.transform(Futures.allAsList(result), list -> { + if (!CollectionUtils.isEmpty(list)) { + return list.stream() + .flatMap(messageList -> { + if (!messageList.isEmpty()) { + this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); + return messageList.stream(); + } + return Stream.empty(); + }) + .collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + } + + @Override + public void commit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + + pendingMessages.clear(); + } + + public T decode(Message message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + + @Data + private static class AwsSqsMsgWrapper { + private final String url; + private final List messages; + + public AwsSqsMsgWrapper(String url, List messages) { + this.url = url; + this.messages = messages; + } + } + + private String getQueueUrl(String topic) { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java new file mode 100644 index 0000000000..2d85539184 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.SendMessageRequest; +import com.amazonaws.services.sqs.model.SendMessageResult; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; + +@Slf4j +public class TbAwsSqsProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final AmazonSQS sqsClient; + private final Gson gson = new Gson(); + private final Map queueUrlMap = new ConcurrentHashMap<>(); + private final TbQueueAdmin admin; + private ListeningExecutorService producerExecutor; + + public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + SendMessageRequest sendMsgRequest = new SendMessageRequest(); + sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); + + sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(SendMessageResult result) { + if (callback != null) { + callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, producerExecutor); + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + + private String getQueueUrl(String topic) { + return queueUrlMap.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java new file mode 100644 index 0000000000..70a9587bef --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.services.sqs.model.QueueAttributeName; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +public class TbAwsSqsQueueAttributes { + @Value("${queue.aws-sqs.queue-properties.core}") + private String coreProperties; + @Value("${queue.aws-sqs.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.aws-sqs.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.aws-sqs.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.aws-sqs.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreAttributes; + @Getter + private Map ruleEngineAttributes; + @Getter + private Map transportApiAttributes; + @Getter + private Map notificationsAttributes; + @Getter + private Map jsExecutorAttributes; + + private final Map defaultAttributes = new HashMap<>(); + + @PostConstruct + private void init() { + defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); + defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); + + coreAttributes = getConfigs(coreProperties); + ruleEngineAttributes = getConfigs(ruleEngineProperties); + transportApiAttributes = getConfigs(transportApiProperties); + notificationsAttributes = getConfigs(notificationsProperties); + jsExecutorAttributes = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + validateAttributeName(key); + configs.put(key, value); + } + configs.putAll(defaultAttributes); + return configs; + } + + private void validateAttributeName(String key) { + QueueAttributeName.fromValue(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java similarity index 50% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java index 343c5e6dd8..922a2b1062 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -13,39 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.sqs; -import lombok.Getter; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; - -/** - * Created by ashvayka on 12.10.18. - */ @Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") @Component -public class TbNodeIdProvider { +@Data +public class TbAwsSqsSettings { + + @Value("${queue.aws_sqs.access_key_id}") + private String accessKeyId; + + @Value("${queue.aws_sqs.secret_access_key}") + private String secretAccessKey; - @Getter - @Value("${cluster.node_id:#{null}}") - private String nodeId; + @Value("${queue.aws_sqs.region}") + private String region; - @PostConstruct - public void init() { - if (StringUtils.isEmpty(nodeId)) { - try { - nodeId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current NodeId: {}", nodeId); - } + @Value("${queue.aws_sqs.threads_per_topic}") + private int threadsPerTopic; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java new file mode 100644 index 0000000000..ca2316e174 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public @interface TbCoreComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java new file mode 100644 index 0000000000..3b99c94dea --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +public @interface TbRuleEngineComponent { +} diff --git a/application/src/main/proto/jsinvoke.proto b/common/queue/src/main/proto/jsinvoke.proto similarity index 100% rename from application/src/main/proto/jsinvoke.proto rename to common/queue/src/main/proto/jsinvoke.proto diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto new file mode 100644 index 0000000000..51bc7649be --- /dev/null +++ b/common/queue/src/main/proto/queue.proto @@ -0,0 +1,410 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package transport; + +option java_package = "org.thingsboard.server.gen.transport"; +option java_outer_classname = "TransportProtos"; + +message QueueInfo { + string name = 1; + string topic = 2; + int32 partitions = 3; +} + +/** + * Service Discovery Data Structures; + */ +message ServiceInfo { + string serviceId = 1; + repeated string serviceTypes = 2; + int64 tenantIdMSB = 3; + int64 tenantIdLSB = 4; + repeated QueueInfo ruleEngineQueues = 5; +} + +/** + * Transport Service Data Structures; + */ +message SessionInfoProto { + string nodeId = 1; + int64 sessionIdMSB = 2; + int64 sessionIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + int64 deviceIdMSB = 6; + int64 deviceIdLSB = 7; + string deviceName = 8; + string deviceType = 9; +} + +enum SessionEvent { + OPEN = 0; + CLOSED = 1; +} + +enum SessionType { + SYNC = 0; + ASYNC = 1; +} + +enum KeyValueType { + BOOLEAN_V = 0; + LONG_V = 1; + DOUBLE_V = 2; + STRING_V = 3; + JSON_V = 4; +} + +message KeyValueProto { + string key = 1; + KeyValueType type = 2; + bool bool_v = 3; + int64 long_v = 4; + double double_v = 5; + string string_v = 6; + string json_v = 7; +} + +message TsKvProto { + int64 ts = 1; + KeyValueProto kv = 2; +} + +message TsKvListProto { + int64 ts = 1; + repeated KeyValueProto kv = 2; +} + +message DeviceInfoProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + string deviceName = 5; + string deviceType = 6; + string additionalInfo = 7; +} + +/** + * Transport Service Messages; + */ +message SessionEventMsg { + SessionType sessionType = 1; + SessionEvent event = 2; +} + +message PostTelemetryMsg { + repeated TsKvListProto tsKvList = 1; +} + +message PostAttributeMsg { + repeated KeyValueProto kv = 1; +} + +message GetAttributeRequestMsg { + int32 requestId = 1; + repeated string clientAttributeNames = 2; + repeated string sharedAttributeNames = 3; +} + +message GetAttributeResponseMsg { + int32 requestId = 1; + repeated TsKvProto clientAttributeList = 2; + repeated TsKvProto sharedAttributeList = 3; + repeated string deletedAttributeKeys = 4; + string error = 5; +} + +message AttributeUpdateNotificationMsg { + repeated TsKvProto sharedUpdated = 1; + repeated string sharedDeleted = 2; +} + +message ValidateDeviceTokenRequestMsg { + string token = 1; +} + +message ValidateDeviceX509CertRequestMsg { + string hash = 1; +} + +message ValidateDeviceCredentialsResponseMsg { + DeviceInfoProto deviceInfo = 1; + string credentialsBody = 2; +} + +message GetOrCreateDeviceFromGatewayRequestMsg { + int64 gatewayIdMSB = 1; + int64 gatewayIdLSB = 2; + string deviceName = 3; + string deviceType = 4; +} + +message GetOrCreateDeviceFromGatewayResponseMsg { + DeviceInfoProto deviceInfo = 1; +} + +message GetTenantRoutingInfoRequestMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; +} + +message GetTenantRoutingInfoResponseMsg { + bool isolatedTbCore = 1; + bool isolatedTbRuleEngine = 2; +} + +message SessionCloseNotificationProto { + string message = 1; +} + +message SubscribeToAttributeUpdatesMsg { + bool unsubscribe = 1; +} + +message SubscribeToRPCMsg { + bool unsubscribe = 1; +} + +message ToDeviceRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToDeviceRpcResponseMsg { + int32 requestId = 1; + string payload = 2; +} + +message ToServerRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToServerRpcResponseMsg { + int32 requestId = 1; + string payload = 2; + string error = 3; +} + +message ClaimDeviceMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + string secretKey = 3; + int64 durationMs = 4; +} + +//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. +message SubscriptionInfoProto { + int64 lastActivityTime = 1; + bool attributeSubscription = 2; + bool rpcSubscription = 3; +} + +message SessionSubscriptionInfoProto { + SessionInfoProto sessionInfo = 1; + SubscriptionInfoProto subscriptionInfo = 2; +} + +message DeviceSessionsCacheEntry { + repeated SessionSubscriptionInfoProto sessions = 1; +} + +message TransportToDeviceActorMsg { + SessionInfoProto sessionInfo = 1; + SessionEventMsg sessionEvent = 2; + GetAttributeRequestMsg getAttributes = 3; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; + SubscribeToRPCMsg subscribeToRPC = 5; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; + SubscriptionInfoProto subscriptionInfo = 7; + ClaimDeviceMsg claimDevice = 8; +} + +message TransportToRuleEngineMsg { + SessionInfoProto sessionInfo = 1; + PostTelemetryMsg postTelemetry = 2; + PostAttributeMsg postAttributes = 3; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; + ToServerRpcRequestMsg toServerRPCCallRequest = 5; +} + +/** + * TB Core Data Structures + */ + +message TbSubscriptionProto { + string serviceId = 1; + string sessionId = 2; + int32 subscriptionId = 3; + string entityType = 4; + int64 tenantIdMSB = 5; + int64 tenantIdLSB = 6; + int64 entityIdMSB = 7; + int64 entityIdLSB = 8; +} + +message TbTimeSeriesSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + int64 startTime = 4; + int64 endTime = 5; +} + +message TbAttributeSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + string scope = 4; +} + +message TbSubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated TbSubscriptionUpdateValueListProto data = 5; +} + +message TbAttributeUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + string scope = 6; + repeated TsKvProto data = 7; +} + +message TbTimeSeriesUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + repeated TsKvProto data = 6; +} + +message TbSubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message TbSubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message TbSubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + +/** + * TB Core to TB Core messages + */ + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message SubscriptionMgrMsgProto { + TbTimeSeriesSubscriptionProto telemetrySub = 1; + TbAttributeSubscriptionProto attributeSub = 2; + TbSubscriptionCloseProto subClose = 3; + TbTimeSeriesUpdateProto tsUpdate = 4; + TbAttributeUpdateProto attrUpdate = 5; +} + +message LocalSubscriptionServiceMsgProto { + TbSubscriptionUpdateProto subUpdate = 1; +} + +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} +/** + * Main messages; + */ + +/* Request from Transport Service to ThingsBoard Core Service */ +message TransportApiRequestMsg { + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; +} + +/* Response from ThingsBoard Core Service to Transport Service */ +message TransportApiResponseMsg { + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; +} + +/* Messages that are handled by ThingsBoard Core Service */ +message ToCoreMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; + DeviceStateServiceMsgProto deviceStateServiceMsg = 2; + SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; + bytes toDeviceActorNotificationMsg = 4; +} + +/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +message ToCoreNotificationMsg { + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; + bytes componentLifecycleMsg = 3; +} + +/* Messages that are handled by ThingsBoard RuleEngine Service */ +message ToRuleEngineMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + bytes tbMsg = 3; + repeated string relationTypes = 4; + string failureMessage = 5; +} + +message ToRuleEngineNotificationMsg { + bytes componentLifecycleMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; +} + +/* Messages that are handled by ThingsBoard Transport Service */ +message ToTransportMsg { + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; +} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 5be2f76258..864e7b2ab9 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 9d2cff5c09..82678fde5d 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,6 +303,8 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index d725d47a5f..fd4c8e2d95 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,14 +20,8 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; -import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -36,7 +30,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 9df658224a..c2f171e39b 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { @@ -221,6 +221,8 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 3d344d8dc5..28c38bdda6 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 107639f302..b058a1c260 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,33 +15,23 @@ */ package org.thingsboard.server.transport.mqtt; -import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 84c7598d9b..7295cb786b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -410,13 +410,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); - transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - } + doDisconnect(); } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -485,9 +479,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { + doDisconnect(); + } + + private void doDisconnect() { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + deviceSessionCtx.setDisconnected(); } } @@ -505,8 +507,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 8f752b5066..fb8a0b70be 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index 299f5449d9..b4b5f98238 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,6 +44,8 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) + .setDeviceName(deviceInfo.getDeviceName()) + .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index e0a944ed60..de5f82f324 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 16bc802cd3..b95bc4e54f 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,6 +36,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common data @@ -99,19 +103,6 @@ org.apache.commons commons-lang3 - - com.google.protobuf - protobuf-java - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index ab67beee0b..c49b9c5bd3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,7 +20,9 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -32,15 +34,16 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -public class TransportContext { +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") +public abstract class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; - @Autowired - private TbNodeIdProvider nodeIdProvider; + private TbServiceInfoProvider serviceInfoProvider; @Getter private ExecutorService executor; @@ -58,7 +61,7 @@ public class TransportContext { } public String getNodeId() { - return nodeIdProvider.getNodeId(); + return serviceInfoProvider.getServiceId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index b0772de945..2af55c9206 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.common.transport; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -35,14 +39,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { + GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); + void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -62,7 +68,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java deleted file mode 100644 index 77c45e1714..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 17.10.18. - */ -@Slf4j -public abstract class AbstractTransportService implements TransportService { - - @Value("${transport.rate_limits.enabled}") - private boolean rateLimitEnabled; - @Value("${transport.rate_limits.tenant}") - private String perTenantLimitsConf; - @Value("${transport.rate_limits.device}") - private String perDevicesLimitsConf; - @Value("${transport.sessions.inactivity_timeout}") - private long sessionInactivityTimeout; - @Value("${transport.sessions.report_timeout}") - private long sessionReportTimeout; - - protected ScheduledExecutorService schedulerExecutor; - protected ExecutorService transportCallbackExecutor; - - private ConcurrentMap sessions = new ConcurrentHashMap<>(); - - //TODO: Implement cleanup of this maps. - private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - - @Override - public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, - TransportServiceCallback callback) { - registerClaimingInfo(sessionInfo, msg, callback); - } - - @Override - public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - reportActivityInternal(sessionInfo); - } - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); - - protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); - - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toId(sessionInfo); - SessionMetaData sessionMetaData = sessions.get(sessionId); - if (sessionMetaData != null) { - sessionMetaData.updateLastActivityTime(); - } - return sessionMetaData; - } - - private void checkInactivityAndReportActivity() { - long expTime = System.currentTimeMillis() - sessionInactivityTimeout; - sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getLastActivityTime() < expTime) { - if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); - } - process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); - sessions.remove(uuid); - sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - } else { - if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { - final long lastActivityTime = sessionMD.getLastActivityTime(); - process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) - .setRpcSubscription(sessionMD.isSubscribedToRPC()) - .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { - @Override - public void onSuccess(Void msg) { - sessionMD.setLastReportedActivityTime(lastActivityTime); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to report last activity time", uuid, e); - } - }); - } - } - }); - } - - @Override - public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { - SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toId(sessionInfo), currentSession); - - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { - listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - deregisterSession(sessionInfo); - }, timeout, TimeUnit.MILLISECONDS); - - currentSession.setScheduledFuture(executorFuture); - } - - @Override - public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toId(sessionInfo)); - if (currentSession != null && currentSession.hasScheduledFuture()) { - log.debug("Stopping scheduler to avoid resending response if request has been ack."); - currentSession.getScheduledFuture().cancel(false); - } - sessions.remove(toId(sessionInfo)); - } - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - if (!rateLimitEnabled) { - return true; - } - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.TENANT)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); - } - return false; - } - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); - } - return false; - } - - return true; - } - - protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { - UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); - SessionMetaData md = sessions.get(sessionId); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - if (toSessionMsg.hasGetAttributesResponse()) { - listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); - } - if (toSessionMsg.hasAttributeUpdateNotification()) { - listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); - } - if (toSessionMsg.hasSessionCloseNotification()) { - listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); - } - if (toSessionMsg.hasToDeviceRequest()) { - listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); - } - if (toSessionMsg.hasToServerResponse()) { - listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); - } - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - //TODO: should we notify the device actor about missed session? - log.debug("[{}] Missing session.", sessionId); - } - } - - protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - } - - protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); - } - - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - } - - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java new file mode 100644 index 0000000000..c820167de1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -0,0 +1,600 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.util.JsonUtils; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by ashvayka on 17.10.18. + */ +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") +public class DefaultTransportService implements TransportService { + + @Value("${transport.rate_limits.enabled}") + private boolean rateLimitEnabled; + @Value("${transport.rate_limits.tenant}") + private String perTenantLimitsConf; + @Value("${transport.rate_limits.device}") + private String perDevicesLimitsConf; + @Value("${transport.sessions.inactivity_timeout}") + private long sessionInactivityTimeout; + @Value("${transport.sessions.report_timeout}") + private long sessionReportTimeout; + @Value("${transport.client_side_rpc.timeout:60000}") + private long clientSideRpcTimeout; + @Value("${queue.transport.poll_interval}") + private int notificationsPollDuration; + + private final Gson gson = new Gson(); + private final TbTransportQueueFactory queueProvider; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; + protected TbQueueProducer> ruleEngineMsgProducer; + protected TbQueueProducer> tbCoreMsgProducer; + protected TbQueueConsumer> transportNotificationsConsumer; + + protected ScheduledExecutorService schedulerExecutor; + protected ExecutorService transportCallbackExecutor; + + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); + //TODO: Implement cleanup of this maps. + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); + private volatile boolean stopped = false; + + public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.queueProvider = queueProvider; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + } + + @PostConstruct + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); + ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); + transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); + transportApiRequestTemplate.init(); + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + if (records.size() == 0) { + continue; + } + records.forEach(record -> { + try { + processToTransportMsg(record.getValue()); + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + transportNotificationsConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + }); + } + + @PreDestroy + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + stopped = true; + + if (transportNotificationsConsumer != null) { + transportNotificationsConsumer.unsubscribe(); + } + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + if (transportApiRequestTemplate != null) { + transportApiRequestTemplate.stop(); + } + } + + @Override + public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { + sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + } + + @Override + public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { + TbProtoQueueMsg protoMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getGetTenantRoutingInfoResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); + } + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build(), callback); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("ts", tsKv.getTs() + ""); + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, packCallback); + } + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build(), callback); + } + } + + private void processTimeout(String requestId) { + RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + SessionMetaData md = sessions.get(data.getSessionId()); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + TransportProtos.ToServerRpcResponseMsg responseMsg = + TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(data.getRequestId()) + .setError("timeout").build(); + listener.onToServerRpcResponse(responseMsg); + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + log.debug("[{}] Missing session.", data.getSessionId()); + } + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + UUID sessionId = toSessionId(sessionInfo); + TenantId tenantId = getTenantId(sessionInfo); + DeviceId deviceId = getDeviceId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", msg.getMethodName()); + json.add("params", JsonUtils.parse(msg.getParams())); + + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("requestId", Integer.toString(msg.getRequestId())); + metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); + metaData.putValue("sessionId", sessionId.toString()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + + String requestId = sessionId + "-" + msg.getRequestId(); + toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); + schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build(), callback); + } + } + + @Override + public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + reportActivityInternal(sessionInfo); + } + + private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { + UUID sessionId = toSessionId(sessionInfo); + SessionMetaData sessionMetaData = sessions.get(sessionId); + if (sessionMetaData != null) { + sessionMetaData.updateLastActivityTime(); + } + return sessionMetaData; + } + + private void checkInactivityAndReportActivity() { + long expTime = System.currentTimeMillis() - sessionInactivityTimeout; + sessions.forEach((uuid, sessionMD) -> { + if (sessionMD.getLastActivityTime() < expTime) { + if (log.isDebugEnabled()) { + log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + } + process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + sessions.remove(uuid); + sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + } else { + if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { + final long lastActivityTime = sessionMD.getLastActivityTime(); + process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) + .setRpcSubscription(sessionMD.isSubscribedToRPC()) + .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { + @Override + public void onSuccess(Void msg) { + sessionMD.setLastReportedActivityTime(lastActivityTime); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to report last activity time", uuid, e); + } + }); + } + } + }); + } + + @Override + public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { + SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); + sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); + + ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + deregisterSession(sessionInfo); + }, timeout, TimeUnit.MILLISECONDS); + + currentSession.setScheduledFuture(executorFuture); + } + + @Override + public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { + SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); + if (currentSession != null && currentSession.hasScheduledFuture()) { + log.debug("Stopping scheduler to avoid resending response if request has been ack."); + currentSession.getScheduledFuture().cancel(false); + } + sessions.remove(toSessionId(sessionInfo)); + } + + @Override + public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); + } + if (!rateLimitEnabled) { + return true; + } + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.TENANT)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); + } + return false; + } + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.DEVICE)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); + } + return false; + } + + return true; + } + + protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { + UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); + SessionMetaData md = sessions.get(sessionId); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + if (toSessionMsg.hasGetAttributesResponse()) { + listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); + } + if (toSessionMsg.hasAttributeUpdateNotification()) { + listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); + } + if (toSessionMsg.hasSessionCloseNotification()) { + listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); + } + if (toSessionMsg.hasToDeviceRequest()) { + listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); + } + if (toSessionMsg.hasToServerResponse()) { + String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); + toServerRpcPendingMap.remove(requestId); + listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); + } + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + //TODO: should we notify the device actor about missed session? + log.debug("[{}] Missing session.", sessionId); + } + } + + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); + } + + protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); + } + + protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { + return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + } + + protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { + return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + } + + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); + } + + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + tbCoreMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); + } + + @Override + public void onFailure(Throwable t) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); + } + } + + private class MsgPackCallback implements TbQueueCallback { + private final AtomicInteger msgCount; + private final TransportServiceCallback callback; + + public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { + this.msgCount = new AtomicInteger(msgCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (msgCount.decrementAndGet() <= 0) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onError(t); + } + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java deleted file mode 100644 index 17031791d5..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ /dev/null @@ -1,339 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Created by ashvayka on 05.10.18. - */ -@ConditionalOnExpression("'${transport.type:null}'=='null'") -@Service -@Slf4j -public class RemoteTransportService extends AbstractTransportService { - - @Value("${kafka.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${kafka.notifications.topic}") - private String notificationsTopic; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - @Value("${kafka.notifications.auto_commit_interval}") - private int notificationsAutoCommitInterval; - @Value("${kafka.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${kafka.transport_api.responses_topic}") - private String transportApiResponsesTopic; - @Value("${kafka.transport_api.max_pending_requests}") - private long maxPendingRequests; - @Value("${kafka.transport_api.max_requests_timeout}") - private long maxRequestsTimeout; - @Value("${kafka.transport_api.response_poll_interval}") - private int responsePollDuration; - @Value("${kafka.transport_api.response_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - //We use this to get the node id. We should replace this with a component that provides the node id. - @Autowired - private TbNodeIdProvider nodeIdProvider; - - private TbKafkaRequestTemplate transportApiTemplate; - private TBKafkaProducerTemplate ruleEngineProducer; - private TBKafkaConsumerTemplate mainConsumer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); - - private volatile boolean stopped = false; - - @PostConstruct - public void init() { - super.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiRequestsTopic); - requestBuilder.encoder(new TransportApiRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-api-client"); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new TransportApiResponseDecoder()); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - transportApiTemplate = builder.build(); - transportApiTemplate.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); - ruleEngineProducerBuilder.settings(kafkaSettings); - ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); - ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); - ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); - ruleEngineProducer = ruleEngineProducerBuilder.build(); - ruleEngineProducer.init(); - - String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); - - try { - TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); - CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - log.trace("Failed to create topic: {}", e.getMessage(), e); - } - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); - mainConsumerBuilder.settings(kafkaSettings); - mainConsumerBuilder.topic(notificationsTopicName); - mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - mainConsumerBuilder.groupId("transport"); - mainConsumerBuilder.autoCommit(true); - mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); - mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); - mainConsumer = mainConsumerBuilder.build(); - mainConsumer.subscribe(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); - records.forEach(record -> { - try { - ToTransportMsg toTransportMsg = mainConsumer.decode(record); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @PreDestroy - public void destroy() { - super.destroy(); - stopped = true; - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), - TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), - TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), - TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { - if (callback != null) { - if (exception == null) { - this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } else { - this.transportCallbackExecutor.submit(() -> { - callback.onError(exception); - }); - } - } - }); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java similarity index 76% rename from common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java index d06046fd71..2c5da0d3ea 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java @@ -13,20 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.core; +package org.thingsboard.server.common.transport.service; import lombok.Data; -/** - * @author Andrew Shvayka - */ -@Data -public class ToServerRpcResponseMsg { +import java.util.UUID; +@Data +public class RpcRequestMetadata { + private final UUID sessionId; private final int requestId; - private final String data; - - public boolean isSuccess() { - return true; - } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index 42a65440a0..fc1445539b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index c07e406d2e..cc154f87b2 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,8 +15,9 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { + @Override - public ToTransportMsg decode(byte[] data) throws IOException { - return ToTransportMsg.parseFrom(data); + public ToTransportMsg decode(TbQueueMsg msg) throws IOException { + return ToTransportMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 6c4b362f03..3971fdbf83 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index a62e696500..5fdc58c068 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,8 +15,9 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { + @Override - public TransportApiResponseMsg decode(byte[] data) throws IOException { - return TransportApiResponseMsg.parseFrom(data); + public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { + return TransportApiResponseMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java new file mode 100644 index 0000000000..f2534a64c9 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { + + private TransportService transportService; + + @Lazy + @Autowired + public void setTransportService(TransportService transportService) { + this.transportService = transportService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index 7c590ec925..c312b3a4d9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,6 +35,7 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; + private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -42,10 +43,15 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; + this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return deviceInfo != null; + return connected; + } + + public void setDisconnected() { + this.connected = false; } } diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonUtils.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java index 5621362c09..583b73dba1 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.transport.util; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/transport/transport-api/src/main/proto/transport.proto deleted file mode 100644 index e8b513574a..0000000000 --- a/common/transport/transport-api/src/main/proto/transport.proto +++ /dev/null @@ -1,244 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package transport; - -option java_package = "org.thingsboard.server.gen.transport"; -option java_outer_classname = "TransportProtos"; - -/** - * Data Structures; - */ -message SessionInfoProto { - string nodeId = 1; - int64 sessionIdMSB = 2; - int64 sessionIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - int64 deviceIdMSB = 6; - int64 deviceIdLSB = 7; -} - -enum SessionEvent { - OPEN = 0; - CLOSED = 1; -} - -enum SessionType { - SYNC = 0; - ASYNC = 1; -} - -enum KeyValueType { - BOOLEAN_V = 0; - LONG_V = 1; - DOUBLE_V = 2; - STRING_V = 3; - JSON_V = 4; -} - -message KeyValueProto { - string key = 1; - KeyValueType type = 2; - bool bool_v = 3; - int64 long_v = 4; - double double_v = 5; - string string_v = 6; - string json_v = 7; -} - -message TsKvProto { - int64 ts = 1; - KeyValueProto kv = 2; -} - -message TsKvListProto { - int64 ts = 1; - repeated KeyValueProto kv = 2; -} - -message DeviceInfoProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - string deviceName = 5; - string deviceType = 6; - string additionalInfo = 7; -} - -/** - * Messages that use Data Structures; - */ -message SessionEventMsg { - SessionType sessionType = 1; - SessionEvent event = 2; -} - -message PostTelemetryMsg { - repeated TsKvListProto tsKvList = 1; -} - -message PostAttributeMsg { - repeated KeyValueProto kv = 1; -} - -message GetAttributeRequestMsg { - int32 requestId = 1; - repeated string clientAttributeNames = 2; - repeated string sharedAttributeNames = 3; -} - -message GetAttributeResponseMsg { - int32 requestId = 1; - repeated TsKvProto clientAttributeList = 2; - repeated TsKvProto sharedAttributeList = 3; - repeated string deletedAttributeKeys = 4; - string error = 5; -} - -message AttributeUpdateNotificationMsg { - repeated TsKvProto sharedUpdated = 1; - repeated string sharedDeleted = 2; -} - -message ValidateDeviceTokenRequestMsg { - string token = 1; -} - -message ValidateDeviceX509CertRequestMsg { - string hash = 1; -} - -message ValidateDeviceCredentialsResponseMsg { - DeviceInfoProto deviceInfo = 1; - string credentialsBody = 2; -} - -message GetOrCreateDeviceFromGatewayRequestMsg { - int64 gatewayIdMSB = 1; - int64 gatewayIdLSB = 2; - string deviceName = 3; - string deviceType = 4; -} - -message GetOrCreateDeviceFromGatewayResponseMsg { - DeviceInfoProto deviceInfo = 1; -} - -message SessionCloseNotificationProto { - string message = 1; -} - -message SubscribeToAttributeUpdatesMsg { - bool unsubscribe = 1; -} - -message SubscribeToRPCMsg { - bool unsubscribe = 1; -} - -message ToDeviceRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToDeviceRpcResponseMsg { - int32 requestId = 1; - string payload = 2; -} - -message ToServerRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToServerRpcResponseMsg { - int32 requestId = 1; - string payload = 2; - string error = 3; -} - -message ClaimDeviceMsg { - int64 deviceIdMSB = 1; - int64 deviceIdLSB = 2; - string secretKey = 3; - int64 durationMs = 4; -} - -//Used to report session state to tb-node and persist this state in the cache on the tb-node level. -message SubscriptionInfoProto { - int64 lastActivityTime = 1; - bool attributeSubscription = 2; - bool rpcSubscription = 3; -} - -message SessionSubscriptionInfoProto { - SessionInfoProto sessionInfo = 1; - SubscriptionInfoProto subscriptionInfo = 2; -} - -message DeviceSessionsCacheEntry { - repeated SessionSubscriptionInfoProto sessions = 1; -} - -message TransportToDeviceActorMsg { - SessionInfoProto sessionInfo = 1; - SessionEventMsg sessionEvent = 2; - PostTelemetryMsg postTelemetry = 3; - PostAttributeMsg postAttributes = 4; - GetAttributeRequestMsg getAttributes = 5; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; - SubscribeToRPCMsg subscribeToRPC = 7; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; - ToServerRpcRequestMsg toServerRPCCallRequest = 9; - SubscriptionInfoProto subscriptionInfo = 10; - ClaimDeviceMsg claimDevice = 11; -} - -message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} - -/** - * Main messages; - */ -message ToRuleEngineMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; -} - -message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; -} - -message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; -} - -message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 8148fccf79..49926ad5c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 5867e505b6..7080722056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index 386220df55..b5bda06a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getId(); + UUID getUuid(); - void setId(UUID id); + void setUuid(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index d75262e311..44fd70f977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,14 +35,15 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getId() { + public UUID getUuid() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - public void setId(UUID id) { + @Override + public void setUuid(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..9f72567838 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,6 +32,9 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units + public static final long EPOCH_DIFF = 122192928000000000L; + /** * Generic constants. */ @@ -112,6 +115,8 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; + public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; + public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java index 5e822a518c..22d8b1c212 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -110,7 +110,7 @@ public abstract class AbstractAlarmEntity extends BaseSqlEntity public AbstractAlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setId(alarm.getId().getId()); + this.setUuid(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index 4ab6144649..fac8d68f62 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -75,7 +75,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity public AbstractAssetEntity(Asset asset) { if (asset.getId() != null) { - this.setId(asset.getId().getId()); + this.setUuid(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 6a973335e1..042fcec950 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -68,7 +68,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti public AbstractDeviceEntity(Device device) { if (device.getId() != null) { - this.setId(device.getId().getId()); + this.setUuid(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -104,8 +104,8 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti } protected Device toDevice() { - Device device = new Device(new DeviceId(getId())); - device.setCreatedTime(UUIDs.unixTimestamp(getId())); + Device device = new Device(new DeviceId(getUuid())); + device.setCreatedTime(UUIDs.unixTimestamp(getUuid())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index c5f0fd36f2..3d140039ce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -94,7 +94,7 @@ public abstract class AbstractEntityViewEntity extends Bas public AbstractEntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setId(entityView.getId().getId()); + this.setUuid(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -145,8 +145,8 @@ public abstract class AbstractEntityViewEntity extends Bas } protected EntityView toEntityView() { - EntityView entityView = new EntityView(new EntityViewId(getId())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); + EntityView entityView = new EntityView(new EntityViewId(getUuid())); + entityView.setCreatedTime(UUIDs.unixTimestamp(getUuid())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); @@ -169,4 +169,5 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setAdditionalInfo(additionalInfo); return entityView; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 21c411d3e2..1b1a27a04f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -56,7 +56,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setId(adminSettings.getId().getId()); + this.setUuid(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index fa9490747a..d05e3cdde1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -19,6 +19,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.alarm.Alarm; + import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Entity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index 4de738ac73..e2573ea0d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setId(auditLog.getId().getId()); + this.setUuid(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(getId())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); + AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index 26946cf207..d5b287fe9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,7 +34,6 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; -import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -72,7 +71,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setId(customer.getId().getId()); + this.setUuid(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(getId())); - customer.setCreatedTime(UUIDs.unixTimestamp(getId())); + Customer customer = new Customer(new CustomerId(this.getUuid())); + customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index b606c35d4e..07cec4d12f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setId(dashboard.getId().getId()); + this.setUuid(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index b1ecb2572b..f26cf21934 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setId(dashboardInfo.getId().getId()); + this.setUuid(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index 1d47c7d63c..ba9d7ce637 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setId(ruleChain.getUuidId()); + this.setUuid(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index f36edc260b..1a7866418a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setId(ruleNode.getUuidId()); + this.setUuid(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 239d3698b4..1da71224f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,6 +72,12 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -82,7 +88,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setId(tenant.getId().getId()); + this.setUuid(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -95,6 +101,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -113,8 +121,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(getId())); - tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); + Tenant tenant = new Tenant(new TenantId(this.getUuid())); + tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); @@ -126,6 +134,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 70593ca610..6957f87961 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setId(userCredentials.getId().getId()); + this.setUuid(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 0100110daa..3af671583e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setId(user.getId().getId()); + this.setUuid(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(getId())); - user.setCreatedTime(UUIDs.unixTimestamp(getId())); + User user = new User(new UserId(this.getUuid())); + user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 8e81b30e8f..0cb69b9d8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setId(widgetType.getId().getId()); + this.setUuid(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 7df6ea9fe7..75ea383090 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setId(widgetsBundle.getId().getId()); + this.setUuid(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index d3ce07f901..edc9c97a9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 2f3c72d01b..076d43b0ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -38,7 +38,7 @@ public interface AlarmRepository extends CrudRepository { @Param("alarmType") String alarmType, Pageable pageable); - @Query("SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + + @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + "RelationEntity re " + "WHERE a.tenantId = :tenantId " + "AND a.id = re.toId AND re.toType = 'ALARM' " + @@ -51,7 +51,21 @@ public interface AlarmRepository extends CrudRepository { "AND (:idOffset IS NULL OR a.id < :idOffset) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))", + countQuery = "SELECT count(a) FROM AlarmEntity a, " + + "RelationEntity re " + + "WHERE a.tenantId = :tenantId " + + "AND a.id = re.toId AND re.toType = 'ALARM' " + + "AND re.relationTypeGroup = 'ALARM' " + + "AND re.relationType = :relationType " + + "AND re.fromId = :affectedEntityId " + + "AND re.fromType = :affectedEntityType " + + "AND (:startId IS NULL OR a.id >= :startId) " + + "AND (:endId IS NULL OR a.id <= :endId) " + + "AND (:idOffset IS NULL OR a.id < :idOffset) " + + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") Page findAlarms(@Param("tenantId") String tenantId, @Param("affectedEntityId") String affectedEntityId, @Param("affectedEntityType") String affectedEntityType, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 002c8969da..56ae6a5000 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.asset; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; @@ -43,7 +42,6 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUIDs; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/19/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index 38abc36940..371a627818 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -32,6 +32,8 @@ import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import javax.persistence.criteria.Predicate; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -39,6 +41,7 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.dao.DaoUtil.endTimeToId; import static org.thingsboard.server.dao.DaoUtil.startTimeToId; +import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @SqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index c634e5a064..d8272e3a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index e281bdeec4..c33dd4f5e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index 67f25c2325..ab4f408da5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.component; import com.datastax.driver.core.utils.UUIDs; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -35,11 +34,9 @@ import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; -import java.util.List; import java.util.Objects; import java.util.Optional; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index e859773bf7..0361c8d8dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.customer; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; @@ -29,12 +28,10 @@ import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.UUID; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index 7b0e6a8882..e693e81ae1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -41,9 +41,6 @@ import java.util.UUID; @SqlDao public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao implements DashboardInfoDao { - @Autowired - private RelationDao relationDao; - @Autowired private DashboardInfoRepository dashboardInfoRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 0d8d87f74e..c409f0a91f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -17,10 +17,13 @@ package org.thingsboard.server.dao.sql.entityview; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -39,7 +42,6 @@ import java.util.Optional; import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Victor Basanets on 8/31/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index 59f0da89e0..8a341e0f81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -69,13 +69,14 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi protected Query getQuery(EventEntity entity, String query) { return entityManager.createNativeQuery(query, EventEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) .setParameter("body", entity.getBody().toString()) .setParameter("entity_id", entity.getEntityId()) .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()); + .setParameter("tenant_id", entity.getTenantId()) + .setParameter("ts", entity.getTs()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index 440c618667..f5a00fe7f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -40,11 +40,11 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { @Override protected EventEntity doProcessSaveOrUpdate(EventEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); + return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index c5a6ea2717..528229bb33 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -141,7 +141,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao - DaoUtil.toPageData(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable))); + DaoUtil.toPageData(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable))); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index d2dbe725cd..3a4cbd84e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.rule; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -30,11 +29,9 @@ import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; -import java.util.List; import java.util.Objects; import java.util.UUID; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; @Slf4j @Component diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 826d60b119..7fc55c5752 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -16,11 +16,9 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -30,11 +28,8 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.dao.util.SqlDao; -import java.util.List; import java.util.Objects; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; - /** * Created by Valerii Sosliuk on 4/30/2017. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 9d39a09404..296730e55f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -30,7 +30,6 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.util.SqlDao; -import java.util.List; import java.util.Objects; import java.util.UUID; @@ -84,6 +83,5 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, DaoUtil.toPageable(pageLink))); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index c24e3d8e27..6f7490277a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -30,7 +29,6 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; -import java.util.List; import java.util.Objects; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index c4ac4e9fd8..fafe05957f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -150,8 +150,8 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq keyId, query.getStartTs(), query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( + PageRequest.of(0, query.getLimit(), + Sort.by(Sort.Direction.fromString( query.getOrderBy()), "ts"))); tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index bf4cf5d9e6..0a2e210648 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( + PageRequest.of(0, query.getLimit(), + Sort.by(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index efac1cfc42..b63e4f7846 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,8 +39,6 @@ import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; -import java.util.List; - import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -125,7 +123,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -139,19 +137,31 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - }; + + @Override + protected void validateUpdate(TenantId tenantId, Tenant tenant) { + Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing tenant!"); + } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); + } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new DataValidationException("Can't update isolatedTbCore property!"); + } + } + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 758aaafb10..697e34930a 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -183,7 +184,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -249,3 +252,4 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 55893fc124..931856e1df 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -183,7 +184,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -249,3 +252,28 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 32e2a78620..bb0a964c13 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3789444106..f83e80931b 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index f02ae6b577..e0244c256e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index af9a4b356f..51f34a08d6 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,3 +60,4 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false +service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index c7dd8e6429..781e0ee01e 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,6 +16,8 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 +service.type=monolith + #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql @@ -33,4 +35,15 @@ spring.datasource.hikari.maximumPoolSize = 50 #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 + +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 + +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index c17b1ddde4..f0facf8cc1 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,6 +19,7 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; +let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -43,6 +44,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); + headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -148,7 +150,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse + value: rawResponse, + headers: headers } ] } diff --git a/pom.xml b/pom.xml index 7cadb38031..a880025e3f 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.1.3.RELEASE - 5.1.5.RELEASE - 5.1.4.RELEASE - 2.1.5.RELEASE - 2.9.0 + 2.2.4.RELEASE + 5.2.2.RELEASE + 5.2.2.RELEASE + 2.2.4.RELEASE + 3.1.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.9.9.3 - 2.9.9 - 2.9.9 + 2.10.2 + 2.10.2 + 2.10.2 2.2.6 - 2.11 - 2.4.2 + 2.13 + 2.6.3 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.37.Final + 4.1.45.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.56 + 1.64 2.0.1 2.5.0 2.5.3 @@ -92,9 +92,15 @@ 4.1.1 2.57 2.7.7 - 1.23 + 1.25 + 1.3.10 + 1.11.747 + 1.84.0 + 3.2.0 1.5.0 1.4.3 + 1.9.4 + 3.2.2 @@ -891,6 +897,21 @@ jts-core ${jts.version} + + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + + + com.google.cloud + google-cloud-pubsub + ${pubsub.client.version} + + + com.microsoft.azure + azure-servicebus + ${azure-servicebus.version} + org.passay passay @@ -901,6 +922,37 @@ uap-java ${ua-parser.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + commons-collections + commons-collections + ${commons-collections.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + org.apache.struts + struts-core + ${struts.version} + + + org.apache.struts + struts-taglib + ${struts.version} + + + org.apache.struts + struts-tiles + ${struts.version} + + diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index ad58bce619..14c3d4b996 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -97,7 +97,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -1645,7 +1644,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1997,12 +1996,24 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getTimeUrlParams(TimePageLink pageLink) { - String urlParams = this.getUrlParams(pageLink); + return getUrlParams(pageLink); + } + + private String getUrlParams(TimePageLink pageLink) { + return getUrlParams(pageLink, "startTime", "endTime"); + } + + private String getUrlParamsTs(TimePageLink pageLink) { + return getUrlParams(pageLink, "startTs", "endTs"); + } + + private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { + String urlParams = "limit={limit}&ascOrder={ascOrder}"; if (pageLink.getStartTime() != null) { - urlParams += "&startTime={startTime}"; + urlParams += "&" + startTime + "={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams += "&endTime={endTime}"; + urlParams += "&" + endTime + "={endTime}"; } return urlParams; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java deleted file mode 100644 index bb0210b1ad..0000000000 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.api; - -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -import java.util.function.Consumer; - -public interface RuleChainTransactionService { - - void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - - void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - - void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); - -} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 34bfe4d3f4..7cab689d13 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -28,11 +29,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { + private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originHost; - private final int originPort; + private final String originServiceId; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 69ce133c5d..baac594a17 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; + +import java.util.UUID; import java.util.function.Consumer; /** @@ -23,8 +25,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReply(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); - void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index f57849fa35..d2e19652a2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,6 +44,4 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); - void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); - } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 38914f4553..9b1edc5f5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -45,27 +44,91 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { + /* + * + * METHODS TO CONTROL THE MESSAGE FLOW + * + */ + + /** + * Indicates that message was successfully processed by the rule node. + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using "Success" relationType. + * + * @param msg + */ + void tellSuccess(TbMsg msg); + + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using specified relationType. + * + * @param msg + * @param relationType + */ void tellNext(TbMsg msg, String relationType); - void tellNext(TbMsg msg, String relationType, Throwable th); - + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using one of specified relationTypes. + * + * @param msg + * @param relationTypes + */ void tellNext(TbMsg msg, Set relationTypes); + /** + * Sends message to the current Rule Node with specified delay in milliseconds. + * Note: this message is not queued and may be lost in case of a server restart. + * + * @param msg + */ void tellSelf(TbMsg msg, long delayMs); - boolean isLocalEntity(EntityId entityId); - + /** + * Notifies Rule Engine about failure to process current message. + * + * @param msg - message + * @param th - exception + */ void tellFailure(TbMsg msg, Throwable th); - void updateSelf(RuleNode self); + /** + * Puts new message to queue for processing by the Root Rule Chain + * + * @param msg - message + */ + void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); + + /** + * Puts new message to custom queue for processing + * + * @param msg - message + */ + void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellFailure(TbMsg msg, String failureMessage); + + void enqueueForTellNext(TbMsg msg, String relationType); + + void enqueueForTellNext(TbMsg msg, Set relationTypes); + + void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - void sendTbMsgToRuleEngine(TbMsg msg); + void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void ack(TbMsg tbMsg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -77,8 +140,17 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); + // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); + /* + * + * METHODS TO PROCESS THE MESSAGES + * + */ + + boolean isLocalEntity(EntityId entityId); + RuleNodeId getSelfId(); TenantId getTenantId(); @@ -129,9 +201,7 @@ public interface TbContext { void logJsEvalFailure(); - String getNodeId(); - - RuleChainTransactionService getRuleChainTransactionService(); + String getServiceId(); EventLoopGroup getSharedEventLoop(); @@ -139,7 +209,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); + @Deprecated RedisTemplate getRedisTemplate(); - String getServerAddress(); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 26f7d14f8b..85fc75139b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} + default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} } diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 1ada70091c..299fdae084 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,8 +35,7 @@ UTF-8 ${basedir}/../.. - 1.11.323 - 1.83.0 + 1.11.747 1.16.0 @@ -99,7 +98,6 @@ com.google.cloud google-cloud-pubsub - ${pubsub.client.version} com.google.api.grpc diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index 6320d5625f..b3344105bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import javax.script.ScriptException; - import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -63,16 +61,18 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), + throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); + } else { + ctx.tellSuccess(msg); } }, - t -> ctx.tellFailure(msg, t) - , ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 3ee2127fde..bdf8c5fe1e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellNext(msg, "Success"), + m -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } @@ -122,7 +122,9 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index 5c92c7536c..3b18c540e8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -187,7 +187,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), + throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -202,7 +204,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), + throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -216,7 +220,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index f009b737e9..c5dc5cc5f4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,13 +117,14 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (filteredAttributes != null && !filteredAttributes.isEmpty()) { + if (!filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } + ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -135,8 +136,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); - ctx.tellNext(updMsg, SUCCESS); + ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 62410e3052..2831e890d6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 2e99e75b53..19d8515455 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,18 +68,19 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); - ctx.tellNext(tbMsg, SUCCESS); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); + ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index a583fe3c74..926c3c25f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -89,10 +89,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - withCallback(save(msg, ctx), aVoid -> { - ctx.tellNext(msg, SUCCESS); - }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) { + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index 59d27018d7..d43ebb26ef 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,11 +74,8 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 37f4e4baee..3cc6074165 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,13 +80,10 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index e16b2f2f0e..46ef675ce1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { + public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { updateGeneratorState(ctx); } @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.tellNext(m, SUCCESS); + ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 0ef9c4d46a..1807c5c4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,15 +65,16 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.tellNext(pendingMsg, SUCCESS); + ctx.enqueueForTellNext(pendingMsg, SUCCESS); } } else { - if(pendingMsgs.size() < config.getMaxPendingMsgs()) { + if (pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); + ctx.ack(msg); } else { - ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index 086e3aded0..dc0eb9f396 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -72,14 +72,13 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } - if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index 59c66dfb8b..e5e84ba0b6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index d0fe593711..07a0508a87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index 486f9fbd84..5dd11c9318 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java new file mode 100644 index 0000000000..b5af3f1563 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "acknowledge", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Acknowledges the incoming message", + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbNodeEmptyConfig" +) +public class TbAckNode implements TbNode { + + EmptyNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.ack(msg); + ctx.tellSuccess(msg); + } + + @Override + public void destroy() { + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java new file mode 100644 index 0000000000..d5048eadbd --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import static org.thingsboard.common.util.DonAsynchron.withCallback; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "checkpoint", + configClazz = TbCheckpointNodeConfiguration.class, + nodeDescription = "transfers the message to another queue", + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbActionNodeCheckPointConfig" +) +public class TbCheckpointNode implements TbNode { + + private TbCheckpointNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); + } + + @Override + public void destroy() { + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java index f49b2bdae8..eb5b8e29d9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java @@ -13,25 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.device; +package org.thingsboard.rule.engine.flow; -import akka.actor.ActorRef; import lombok.Data; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.rule.engine.api.NodeConfiguration; -/** - * Created by ashvayka on 15.03.18. - */ @Data -public final class DeviceActorToRuleEngineMsg implements TbActorMsg { +public class TbCheckpointNodeConfiguration implements NodeConfiguration { - private final ActorRef callbackRef; - private final TbMsg tbMsg; + private String queueName; @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_TO_RULE_ENGINE_MSG; + public TbCheckpointNodeConfiguration defaultConfiguration() { + TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); + configuration.setQueueName("HighPriority"); + return configuration; } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 8e3bcdb7dc..5a8b92cf80 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index fe765abdf9..832a86b5cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,11 +47,12 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig") + configDirective = "tbActionNodeGpsGeofencingConfig" +) public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -66,7 +67,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -78,17 +79,26 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } else { + if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } } } + if (!told) { + ctx.tellSuccess(msg); + } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { @@ -108,7 +118,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 267e8711aa..4955ebc400 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -17,12 +17,21 @@ package org.thingsboard.rule.engine.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; -import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -30,7 +39,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Properties; -import java.util.concurrent.ExecutionException; @Slf4j @RuleNode( @@ -63,7 +71,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); @@ -73,8 +81,7 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties() - .forEach(properties::put); + config.getOtherProperties().forEach(properties::put); } addMetadataKeyValuesAsKafkaHeaders = BooleanUtils.toBooleanDefaultIfNull(config.isAddMetadataKeyValuesAsKafkaHeaders(), false); toBytesCharset = config.getKafkaHeadersCharset() != null ? Charset.forName(config.getKafkaHeadersCharset()) : StandardCharsets.UTF_8; @@ -86,7 +93,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { if (!addMetadataKeyValuesAsKafkaHeaders) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index 1c41c2a124..ee46b81116 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -79,7 +78,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellNext(msg, SUCCESS), + ok -> ctx.tellSuccess(msg), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 0bf1c23cc7..8aa15a4f38 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -43,7 +43,6 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -101,7 +100,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index 2906d3922f..43738fd706 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -59,7 +59,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), + ctx::tellSuccess, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index bc19d6c56f..c7387a5ed0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -51,8 +51,7 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback( - findEntityAsync(ctx, msg.getOriginator()), + withCallback(findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -89,7 +88,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 653b571b4b..03f0652c3f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "device attributes", + name = "related device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 717cb207c4..12577076d2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -54,10 +54,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index caef71a4db..a3d84b25db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,8 +106,7 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - ctx.tellNext(newMsg, SUCCESS); + ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 6d45a3a05c..7e27ec496e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellNext(msg, TbRelationTypes.SUCCESS); + ctx.tellSuccess(msg); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 3c3bde2915..4a61cf63f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + ctx::tellSuccess, t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 65eea0b4c9..ba860c103b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -84,7 +83,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + void processMessage(TbContext ctx, TbMsg msg) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -95,13 +94,6 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { - if (config.isUseRedisQueueForMsgPersistence()) { - if (throwable instanceof HttpClientErrorException) { - processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); - } else { - queueProcessor.pushOnFailure(msg); - } - } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -109,15 +101,9 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { - if (config.isUseRedisQueueForMsgPersistence()) { - queueProcessor.resetCounter(); - } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } else { - if (config.isUseRedisQueueForMsgPersistence()) { - processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); - } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -183,11 +169,4 @@ class TbHttpClient { } } - private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { - if (statusCode.is4xxClientError()) { - log.warn("[{}] Client error during message delivering!", msg); - } else { - queueProcessor.pushOnFailure(msg); - } - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java deleted file mode 100644 index cff9faaab5..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.rest; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.redis.core.ListOperations; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Data -@Slf4j -class TbRedisQueueProcessor { - - private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; - - private final TbContext ctx; - private final TbHttpClient httpClient; - private final ExecutorService executor; - private final ListOperations listOperations; - private final String redisKey; - private final boolean trimQueue; - private final int maxQueueSize; - - private AtomicInteger failuresCounter; - private Future future; - - TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { - this.ctx = ctx; - this.httpClient = httpClient; - this.executor = Executors.newSingleThreadExecutor(); - this.listOperations = ctx.getRedisTemplate().opsForList(); - this.redisKey = constructRedisKey(); - this.trimQueue = trimQueue; - this.maxQueueSize = maxQueueSize; - init(); - } - - private void init() { - failuresCounter = new AtomicInteger(0); - future = executor.submit(() -> { - while (true) { - if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { - sleep("Target HTTP server is down...", 3); - } - if (listOperations.size(redisKey) > 0) { - List list = listOperations.range(redisKey, -10, -1); - list.forEach(obj -> { - TbMsg msg = TbMsg.fromBytes((byte[]) obj); - log.debug("Trying to send the message: {}", msg); - listOperations.remove(redisKey, -1, obj); - httpClient.processMessage(ctx, msg, this); - }); - } else { - sleep("Queue is empty, waiting for tasks!", 1); - } - } - }); - } - - void destroy() { - if (future != null) { - future.cancel(true); - } - if (executor != null) { - executor.shutdownNow(); - } - } - - void push(TbMsg msg) { - listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); - if (trimQueue) { - listOperations.trim(redisKey, 0, validateMaxQueueSize()); - } - } - - void pushOnFailure(TbMsg msg) { - listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); - failuresCounter.incrementAndGet(); - } - - void resetCounter() { - failuresCounter.set(0); - } - - private String constructRedisKey() { - return ctx.getServerAddress() + ctx.getSelfId(); - } - - private int validateMaxQueueSize() { - if (maxQueueSize != 0) { - return maxQueueSize; - } - return MAX_QUEUE_SIZE; - } - - private void sleep(String logMessage, int sleepSeconds) { - try { - log.debug(logMessage); - TimeUnit.SECONDS.sleep(sleepSeconds); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted!", e); - } - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9b9d275977..9cb171d0dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,8 +25,6 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.concurrent.ExecutionException; - @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -47,7 +45,6 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; - private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -55,20 +52,13 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - if (ctx.getRedisTemplate() == null) { - throw new RuntimeException("Redis cache type must be used!"); - } - queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); + log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - if (useRedisQueueForMsgPersistence) { - queueProcessor.push(msg); - } else { - httpClient.processMessage(ctx, msg, null); - } + public void onMsg(TbContext ctx, TbMsg msg) { + httpClient.processMessage(ctx, msg); } @Override @@ -76,9 +66,6 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } - if (this.queueProcessor != null) { - this.queueProcessor.destroy(); - } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 9444b52210..145e73d450 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.UUID; + @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -50,15 +52,22 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { + String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); + String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); + } else if (StringUtils.isEmpty(serviceIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); + } else if (StringUtils.isEmpty(sessionIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); + ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 9aa94905e6..ad1d399e76 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -38,7 +36,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -86,10 +83,8 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originHost"); - String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; - tmp = msg.getMetaData().getValue("originPort"); - int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; + tmp = msg.getMetaData().getValue("originServiceId"); + String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -106,24 +101,25 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) + .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originHost(originHost) - .originPort(originPort) + .originServiceId(originServiceId) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); } }); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index d01f28f136..bd3a3cdfb2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,18 +16,40 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; +import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { + public static final String SERVICE_ID = "serviceId"; + public static final String SESSION_ID = "sessionId"; + public static final String REQUEST_ID = "requestId"; + + private String serviceIdMetaDataAttribute; + private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setRequestIdMetaDataAttribute("requestId"); + configuration.setServiceIdMetaDataAttribute(SERVICE_ID); + configuration.setSessionIdMetaDataAttribute(SESSION_ID); + configuration.setRequestIdMetaDataAttribute(REQUEST_ID); return configuration; } + + public String getServiceIdMetaDataAttribute() { + return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; + } + + public String getSessionIdMetaDataAttribute() { + return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; + } + + public String getRequestIdMetaDataAttribute() { + return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; + } } + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 3ad89bc05a..a0970238d9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,13 +61,9 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } - String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); - if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { - ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); - } } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index d735f7f93f..bc135caa5e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap == null) { + if (tsKvMap.isEmpty()) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index d545d41d7b..bd15c5bd83 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; - /** * Created by ashvayka on 02.04.18. */ @@ -34,7 +32,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 5423236499..751b061666 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,10 +25,6 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgTransactionData; - -import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -37,37 +33,23 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Starts synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") +@Deprecated public class TbSynchronizationBeginNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); - - TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); - TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); - - ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { - log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); - ctx.tellNext(startMsg, SUCCESS); - }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), - throwable -> { - log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); - ctx.tellFailure(tbMsg, throwable); - }); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index af3742a0fe..8b83bff1fa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,30 +35,25 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Stops synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) +@Deprecated public class TbSynchronizationEndNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - ctx.getRuleChainTransactionService().endTransaction(msg, - successMsg -> ctx.tellNext(successMsg, SUCCESS), - throwable -> ctx.tellFailure(msg, throwable)); - log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override public void destroy() { - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index e768f699cd..84a6b76920 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellNext(m, SUCCESS); + ctx.tellSuccess(m); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 40a3e8effd..31fd207c7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index a0a1c8629f..182a000b59 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 3ff25e1e8b..017bacfd72 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 67c5b1cc18..61b83d647d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,12 +25,15 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -38,20 +41,35 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; +import java.util.function.Consumer; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { @@ -66,6 +84,11 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; + @Captor + private ArgumentCaptor successCaptor; + @Captor + private ArgumentCaptor> failureCaptor; + private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -99,15 +122,16 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); - doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -141,7 +165,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -164,7 +188,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -175,6 +199,8 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -209,7 +235,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -256,7 +282,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index 27feed93ba..b0f64fb980 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,7 +19,6 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -28,17 +27,24 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { @@ -58,7 +64,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -71,7 +77,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -84,7 +90,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 1e752b55b8..9286a06da6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,10 +28,14 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -40,7 +44,9 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { @@ -65,7 +71,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index ace02bacd7..70f58c5989 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,13 +30,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) @@ -62,7 +62,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0af1cef8ae..a0a61ae30f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,9 +31,19 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -102,7 +112,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -127,7 +137,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -152,7 +162,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +176,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); entityAttributeFetched(customerId); } @@ -177,7 +187,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -192,7 +202,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -207,7 +217,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -234,7 +244,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -246,7 +256,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -258,7 +268,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 4c1edce8f6..fa845c4a18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -91,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -119,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -146,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index aa9ad8b76f..a00e97cf1c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,10 +27,14 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -38,7 +42,10 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) @@ -62,15 +69,15 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); + TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); + verify(ctx).tellSuccess(captor.capture()); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } @@ -84,7 +91,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); diff --git a/tools/pom.xml b/tools/pom.xml index c8058d1bd7..a3018e40ce 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.4 + 3.11.6 com.datastax.cassandra diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index 6dbca324e1..ec596db10d 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 9b3572c7d1..c3dcb76508 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,24 +41,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index 39dc0677e8..f44af2105e 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 4097bce831..baac6e977f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,24 +42,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index ce95ee6f1f..cd7225ad1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index d13a3d071e..e920891d66 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # MQTT server parameters transport: @@ -62,24 +72,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index 761f04989d..db477c3a8c 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -28,6 +28,7 @@ export * from './entity-relation.service'; export * from './entity-view.service'; export * from './event.service'; export * from './http-utils'; +export * from './queue.service'; export * from './rule-chain.service'; export * from './tenant.service'; export * from './user.service'; diff --git a/ui-ngx/src/app/core/http/queue.service.ts b/ui-ngx/src/app/core/http/queue.service.ts new file mode 100644 index 0000000000..42beadb81a --- /dev/null +++ b/ui-ngx/src/app/core/http/queue.service.ts @@ -0,0 +1,36 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable } from 'rxjs'; +import { ServiceType } from '@shared/models/queue.models'; + +@Injectable({ + providedIn: 'root' +}) +export class QueueService { + + constructor( + private http: HttpClient + ) { } + + public getTenantQueuesByServiceType(serviceType: ServiceType, config?: RequestConfig): Observable> { + return this.http.get>(`/api/tenant/queues?serviceType=${serviceType}`, + defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html index 48c57184ab..7d0ccd5056 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html @@ -56,6 +56,16 @@ +
+ +
{{ 'tenant.isolated-tb-core' | translate }}
+
{{'tenant.isolated-tb-core-details' | translate}}
+ + +
{{ 'tenant.isolated-tb-rule-engine' | translate }}
+
{{'tenant.isolated-tb-rule-engine-details' | translate}}
+ +
diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss new file mode 100644 index 0000000000..b05c6324fa --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .mat-checkbox.hinted-checkbox { + .mat-checkbox-inner-container { + margin-top: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts index c8317a20c5..f624a5d5cf 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts @@ -26,7 +26,8 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod @Component({ selector: 'tb-tenant', - templateUrl: './tenant.component.html' + templateUrl: './tenant.component.html', + styleUrls: ['./tenant.component.scss'] }) export class TenantComponent extends ContactBasedComponent { @@ -50,6 +51,8 @@ export class TenantComponent extends ContactBasedComponent { return this.fb.group( { title: [entity ? entity.title : '', [Validators.required]], + isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], + isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], additionalInfo: this.fb.group( { description: [entity && entity.additionalInfo ? entity.additionalInfo.description : ''] @@ -61,6 +64,8 @@ export class TenantComponent extends ContactBasedComponent { updateEntityForm(entity: Tenant) { this.entityForm.patchValue({title: entity.title}); + this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); + this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); this.entityForm.patchValue({additionalInfo: {description: entity.additionalInfo ? entity.additionalInfo.description : ''}}); } diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html new file mode 100644 index 0000000000..ccdf4a76ec --- /dev/null +++ b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html @@ -0,0 +1,43 @@ + + + {{ 'queue.name' | translate }} + + + + + + + + + {{ 'queue.name_required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts new file mode 100644 index 0000000000..4e008d5c6c --- /dev/null +++ b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts @@ -0,0 +1,177 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable } from 'rxjs'; +import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { QueueService } from '@core/http/queue.service'; +import { ServiceType } from '@shared/models/queue.models'; + +@Component({ + selector: 'tb-queue-type-list', + templateUrl: './queue-type-list.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => QueueTypeListComponent), + multi: true + }] +}) +export class QueueTypeListComponent implements ControlValueAccessor, OnInit, AfterViewInit, OnDestroy { + + queueFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + @Input() + queueType: ServiceType; + + @ViewChild('queueInput', {static: true}) queueInput: ElementRef; + + filteredQueues: Observable>; + + queues: Observable>; + + searchText = ''; + + private dirty = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + public translate: TranslateService, + private queueService: QueueService, + private fb: FormBuilder) { + this.queueFormGroup = this.fb.group({ + queue: [null] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.filteredQueues = this.queueFormGroup.get('queue').valueChanges + .pipe( + tap(value => { + this.updateView(value); + }), + map(value => value ? value : ''), + mergeMap(queue => this.fetchQueues(queue) ) + ); + } + + ngAfterViewInit(): void { + } + + ngOnDestroy(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.queueFormGroup.disable({emitEvent: false}); + } else { + this.queueFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + this.queueFormGroup.get('queue').patchValue(value, {emitEvent: false}); + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.queueFormGroup.get('queue').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + updateView(value: string | null) { + if (this.modelValue !== value) { + this.modelValue = value; + this.propagateChange(this.modelValue); + } + } + + displayQueueFn(queue?: string): string | undefined { + return queue ? queue : undefined; + } + + fetchQueues(searchText?: string): Observable> { + this.searchText = searchText; + return this.getQueues().pipe( + map(queues => { + const result = queues.filter( queue => { + return searchText ? queue.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + } else if (searchText && searchText.length) { + result.push(searchText); + } + return result; + }) + ); + } + + getQueues(): Observable> { + if (!this.queues) { + this.queues = this.queueService. + getTenantQueuesByServiceType(this.queueType, {ignoreLoading: true}).pipe( + publishReplay(1), + refCount() + ); + } + return this.queues; + } + + clear() { + this.queueFormGroup.get('queue').patchValue(null, {emitEvent: true}); + setTimeout(() => { + this.queueInput.nativeElement.blur(); + this.queueInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 856c994943..cbc030a4fa 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -36,6 +36,7 @@ export * from './error.models'; export * from './event.models'; export * from './login.models'; export * from './material.models'; +export * from './queue.models'; export * from './relation.models'; export * from './rule-chain.models'; export * from './rule-node.models'; diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts new file mode 100644 index 0000000000..ff8f386dbe --- /dev/null +++ b/ui-ngx/src/app/shared/models/queue.models.ts @@ -0,0 +1,22 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export enum ServiceType { + TB_CORE = 'TB_CORE', + TB_RULE_ENGINE = 'TB_RULE_ENGINE', + TB_TRANSPORT = 'TB_TRANSPORT', + JS_EXECUTOR = 'JS_EXECUTOR' +} diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 266981db1f..4f2e01ea81 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -21,5 +21,7 @@ import {TenantId} from './id/tenant-id'; export interface Tenant extends ContactBased { title: string; region: string; + isolatedTbCore: boolean; + isolatedTbRuleEngine: boolean; additionalInfo?: any; } diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 5008f10459..e94c8a6641 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -129,6 +129,7 @@ import { JsonObjectEditDialogComponent } from '@shared/components/dialog/json-ob import { HistorySelectorComponent } from './components/time/history-selector/history-selector.component'; import { TbTemplatePipe } from '@shared/pipe/template.pipe'; import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-gateway-select.component'; +import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list.component'; @NgModule({ providers: [ @@ -179,6 +180,7 @@ import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-g EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, + QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, @@ -297,6 +299,7 @@ import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-g EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, + QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3112cba1af..2a179b3798 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1514,6 +1514,11 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1541,7 +1546,12 @@ "no-tenants-matching": "No tenants matching '{{entity}}' were found.", "tenant-required": "Tenant is required", "search": "Search tenants", - "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected" + "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected", + "tenant-required": "Tenant is required", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json index 514cd9a8e3..0d6c9abdc9 100644 --- a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json +++ b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json @@ -1404,6 +1404,11 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json index 92e8dc4a99..fa0da05e0a 100644 --- a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json +++ b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json @@ -1970,6 +1970,11 @@ "no-timezones-matching": "Не знайдено жодних часових поясів, які відповідають '{{timezone}}'.", "timezone-required": "Необхідно вказати часовий пояс." }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + }, "tenant": { "tenant": "Власник", "tenants": "Власники", diff --git a/ui/package-lock.json b/ui/package-lock.json index 6a9997cbf0..bd0e0cc260 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" + "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" + "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", + "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", + "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", + "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", + "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", + "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", + "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", "dev": true }, "attr-accept": { @@ -2102,7 +2102,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2126,7 +2126,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2135,7 +2135,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2144,7 +2144,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2290,7 +2290,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2514,7 +2514,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2545,7 +2545,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2722,7 +2722,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3049,7 +3049,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3109,7 +3109,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", "dev": true }, "convert-source-map": { @@ -3136,7 +3136,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3355,7 +3355,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", + "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3483,7 +3483,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "dev": true, "requires": { "ms": "2.0.0" @@ -3561,7 +3561,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3571,7 +3571,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3580,7 +3580,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3589,7 +3589,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3652,7 +3652,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" }, "delegates": { "version": "1.0.0", @@ -3808,7 +3808,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", "dev": true }, "domelementtype": { @@ -3972,7 +3972,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", "dev": true, "requires": { "prr": "~1.0.1" @@ -4491,7 +4491,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4706,7 +4706,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5134,7 +5134,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", + "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", "dev": true }, "fs-write-stream-atomic": { @@ -5175,8 +5175,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5197,14 +5196,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5219,20 +5216,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5349,8 +5343,7 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5362,7 +5355,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5377,7 +5369,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5385,14 +5376,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5411,7 +5400,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5501,8 +5489,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5514,7 +5501,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5600,8 +5586,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5637,7 +5622,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5657,7 +5641,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5701,14 +5684,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -5727,7 +5708,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", "dev": true }, "functional-red-black-tree": { @@ -6681,7 +6662,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6731,7 +6712,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6833,7 +6814,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", "dev": true }, "is-callable": { @@ -6877,7 +6858,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6888,7 +6869,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", "dev": true } } @@ -6996,7 +6977,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7063,7 +7044,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", "dev": true }, "is-word-character": { @@ -7203,7 +7184,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", "dev": true }, "json-schema": { @@ -7220,7 +7201,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7460,7 +7441,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7993,7 +7974,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" + "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" }, "methods": { "version": "1.1.2", @@ -8035,7 +8016,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", "dev": true }, "mime-db": { @@ -8056,7 +8037,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" }, "min-document": { "version": "2.19.0", @@ -8113,7 +8094,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", "requires": { "brace-expansion": "^1.1.7" } @@ -8208,7 +8189,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8286,7 +8267,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8444,7 +8425,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", + "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8504,7 +8485,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8523,7 +8504,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8724,7 +8705,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9041,7 +9022,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9469,7 +9450,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9713,7 +9694,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", "dev": true }, "process": { @@ -9736,7 +9717,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", "requires": { "asap": "~2.0.3" } @@ -9816,7 +9797,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -9981,7 +9962,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", + "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -10012,7 +9993,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", + "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10171,7 +10152,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10183,7 +10164,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", + "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", "requires": { "lodash": "^4.0.1" } @@ -10366,7 +10347,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" }, "regenerator-transform": { "version": "0.14.1", @@ -10380,7 +10361,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10685,7 +10666,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", "dev": true }, "retry": { @@ -10789,7 +10770,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sass-graph": { "version": "2.2.4", @@ -11123,7 +11104,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", "dev": true, "requires": { "base": "^0.11.1", @@ -11159,7 +11140,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11179,7 +11160,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11188,7 +11169,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11197,7 +11178,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11210,7 +11191,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11230,7 +11211,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", + "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11354,7 +11335,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11448,7 +11429,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11588,7 +11569,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11632,7 +11613,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "requires": { "safe-buffer": "~5.1.0" } @@ -12317,7 +12298,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" + "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" }, "table": { "version": "5.4.6", @@ -12547,7 +12528,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", "requires": { "os-tmpdir": "~1.0.2" } @@ -12587,7 +12568,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", "dev": true, "requires": { "define-property": "^2.0.2", @@ -12789,7 +12770,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", "dev": true } } @@ -13114,7 +13095,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13176,7 +13157,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", "dev": true }, "util": { @@ -13941,7 +13922,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", + "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", "dev": true }, "whatwg-fetch": { @@ -14055,7 +14036,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", + "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js new file mode 100644 index 0000000000..0fe93e5fa6 --- /dev/null +++ b/ui/src/app/api/queue.service.js @@ -0,0 +1,40 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default angular.module('thingsboard.api.queue', []) + .factory('queueService', queueService) + .name; + +/*@ngInject*/ +function queueService($http, $q) { + var service = { + getTenantQueuesByServiceType: getTenantQueuesByServiceType + }; + + return service; + + function getTenantQueuesByServiceType(serviceType, config) { + let deferred = $q.defer(); + let url = '/api/tenant/queues?serviceType=' + serviceType; + + $http.get(url, config).then(function success(data) { + deferred.resolve(data); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } +} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js new file mode 100644 index 0000000000..5236603270 --- /dev/null +++ b/ui/src/app/components/queue/index.js @@ -0,0 +1,23 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import thingsboardApiQueue from '../../api/queue.service'; +import queueTypeList from "./queue-type-list.directive"; + +export default angular.module('thingsboard.queue', [ + thingsboardApiQueue +]) + .directive('tbQueueTypeList', queueTypeList) + .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js new file mode 100644 index 0000000000..6a2f99d280 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.directive.js @@ -0,0 +1,111 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import './queue-type-list.scss'; + +/* eslint-disable import/no-unresolved, import/default */ + +import queueTypeListTemplate from './queue-type-list.tpl.html'; + +/* eslint-enable import/no-unresolved, import/default */ + +/*@ngInject*/ +export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { + + var linker = function (scope, element, attrs, ngModelCtrl) { + var template = $templateCache.get(queueTypeListTemplate); + element.html(template); + + scope.queues = null; + scope.queue = null; + scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; + scope.queueSearchText = ''; + + scope.fetchQueues = function(searchText) { + var deferred = $q.defer(); + loadQueues().then( + function success(queueArr) { + let result = $filter('filter')(queueArr, {'$': searchText}); + if (result && result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + deferred.resolve(result); + } else { + deferred.resolve([searchText]); + } + }, + function fail() { + deferred.reject(); + } + ); + + return deferred.promise; + }; + + scope.updateView = function () { + if (!scope.disabled) { + ngModelCtrl.$setViewValue(scope.queue); + } + }; + + function loadQueues() { + var deferred = $q.defer(); + if (!scope.queues) { + queueService.getTenantQueuesByServiceType(scope.queueType).then( + function success(queueArr) { + scope.queues = queueArr.data; + deferred.resolve(scope.queues); + }, + function fail() { + deferred.reject(); + } + ); + } else { + deferred.resolve(scope.queues); + } + return deferred.promise; + } + + ngModelCtrl.$render = function () { + scope.queue = ngModelCtrl.$viewValue; + }; + + scope.$watch('queue', function (newValue, prevValue) { + if (!angular.equals(newValue, prevValue)) { + scope.updateView(); + } + }); + + scope.$watch('disabled', function () { + scope.updateView(); + }); + + $compile(element.contents())(scope); + }; + + return { + restrict: "E", + require: "^ngModel", + link: linker, + scope: { + theForm: '=?', + tbRequired: '=?', + disabled:'=ngDisabled', + queueType: '=?' + } + }; +} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss new file mode 100644 index 0000000000..b8f6f58fb2 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.scss @@ -0,0 +1,15 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html new file mode 100644 index 0000000000..5fc700026e --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.tpl.html @@ -0,0 +1,43 @@ + + + +
+ {{item}} +
+ +
+
{{'queue.name_required' | translate}}
+
+ \ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index a00baa5199..f3b4d10e4f 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,6 +55,7 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; +import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -86,6 +87,7 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, + thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index 8c7a89580d..fa4689fc75 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1489,6 +1489,12 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1514,7 +1520,11 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required" + "tenant-required": "Tenant is required", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index f86eb988ff..e93c92124c 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,6 +1403,12 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index 5a0899014d..c19049bc11 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,6 +1820,12 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index aa7751d479..b75ea30f06 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,32 +15,50 @@ limitations under the License. --> -{{ 'tenant.manage-tenant-admins' | translate }} -{{ 'tenant.delete' | translate }} +{{ + 'tenant.manage-tenant-admins' | translate }} + +{{ 'tenant.delete' + | translate }} +
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
- - - - - - -
+
+ + + +
+
tenant.title-required
+
+ + + + + + + + + {{'tenant.isolated-tb-core' | translate}}
+ {{'tenant.isolated-tb-core-details' | translate}} + + + + + {{'tenant.isolated-tb-rule-engine' | translate}}
+ {{'tenant.isolated-tb-rule-engine-details' | translate}} + + +
From 241edc1d2b8ec51f3e678d7857539086fb017cf1 Mon Sep 17 00:00:00 2001 From: ArtemHalushko <61501795+ArtemHalushko@users.noreply.github.com> Date: Fri, 24 Apr 2020 15:48:00 +0300 Subject: [PATCH 088/102] Map/3.0 (#2664) * add base map infrastructure * add leaflet css * add tencent map * add google maps support * added image map support * refactor schemes * here maps support && WIP on markers * add simple marker suppor * data update & polyline support * map bouds support * add some settings support * add map provider select to settings * labels support * WIP on trip animation widget * WIP on history control and route interpolation * trip-animation map provider & custom markers * comleted track marker & history controls * add license headers * label fix & tooltips support * WIP on polygons * marker dropping support * add polygon support * add label to trip animation * WIP on tooltips * lint anf typed leaflet AddMarker * some typing and poly improvements * add typing * add marker creation * update proxy * save position fix * add bounds padding * update map widget bendle && bugfixes * update marker placement widget * add licenses * reomove log * fix sizes * entity and map fixes * Fix tile server support form OSM and zoom level fix * add support custom actions * map fixes Co-authored-by: Artem Halushko Co-authored-by: Adsumus --- ui-ngx/angular.json | 4 +- ui-ngx/package-lock.json | 14 +- ui-ngx/package.json | 1 + ui-ngx/src/app/core/utils.ts | 59 +++--- .../components/widget/lib/maps/leaflet-map.ts | 83 ++++++-- .../components/widget/lib/maps/map-models.ts | 37 ++-- .../widget/lib/maps/map-widget.interface.ts | 5 +- .../components/widget/lib/maps/map-widget2.ts | 78 ++++++-- .../components/widget/lib/maps/maps-utils.ts | 25 ++- .../components/widget/lib/maps/markers.scss | 4 + .../components/widget/lib/maps/markers.ts | 49 ++--- .../widget/lib/maps/providers/here-map.ts | 7 +- .../components/widget/lib/maps/schemes.ts | 153 +++++++------- .../trip-animation.component.html | 10 +- .../trip-animation.component.scss | 8 +- .../trip-animation.component.ts | 42 ++-- .../history-selector.component.html | 23 ++- .../history-selector.component.scss | 186 +++++++++--------- .../history-selector.component.ts | 4 +- ui-ngx/src/app/shared/pipe/template.pipe.ts | 4 +- 20 files changed, 480 insertions(+), 316 deletions(-) diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index 9d00aa421d..0bdb380c66 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -39,7 +39,9 @@ "node_modules/rc-select/assets/index.less", "node_modules/jstree-bootstrap-theme/dist/themes/proton/style.min.css", "node_modules/leaflet/dist/leaflet.css", - "src/app/modules/home/components/widget/lib/maps/markers.scss" + "src/app/modules/home/components/widget/lib/maps/markers.scss", + "node_modules/leaflet.markercluster/dist/MarkerCluster.css", + "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css" ], "stylePreprocessorOptions": { "includePaths": [ diff --git a/ui-ngx/package-lock.json b/ui-ngx/package-lock.json index 30d331acfb..aec8d12a34 100644 --- a/ui-ngx/package-lock.json +++ b/ui-ngx/package-lock.json @@ -1816,8 +1816,7 @@ "@types/geojson": { "version": "7946.0.7", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.7.tgz", - "integrity": "sha512-wE2v81i4C4Ol09RtsWFAqg3BUitWbHSpSlIo+bNdsCJijO9sjme+zm+73ZMCa/qMC8UEERxzGbvmr1cffo2SiQ==", - "dev": true + "integrity": "sha512-wE2v81i4C4Ol09RtsWFAqg3BUitWbHSpSlIo+bNdsCJijO9sjme+zm+73ZMCa/qMC8UEERxzGbvmr1cffo2SiQ==" }, "@types/glob": { "version": "7.1.1", @@ -1881,7 +1880,6 @@ "version": "1.5.12", "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.5.12.tgz", "integrity": "sha512-61HRMIng+bWvnnAIqUWLBlrd/TQZc4gU+gN1JL4K47EDtwIrcMEhWgi7PdcpbG1YmpH4F0EfOimkvV82gJIl9w==", - "dev": true, "requires": { "@types/geojson": "*" } @@ -1895,6 +1893,14 @@ "@types/leaflet": "*" } }, + "@types/leaflet.markercluster": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@types/leaflet.markercluster/-/leaflet.markercluster-1.4.2.tgz", + "integrity": "sha512-QQ//hevAxMH2dlRQdRre7V/1G+TbtuDtZnZF/75TNwVIgklrsQVCIcS/cvLsl7UUryfPJ6xmoYHfFzK5iGVgpg==", + "requires": { + "@types/leaflet": "*" + } + }, "@types/lodash": { "version": "4.14.150", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.150.tgz", @@ -8605,7 +8611,7 @@ "integrity": "sha512-4O3GWAYJaauMCILm07weko2rHA8a4kjn7+8Lg4s1d7SxwS/3IpkVD/GljbRrIJ1c1W/XGJ3GbuK7RyYZEJChhw==" }, "ngx-flowchart": { - "version": "git://github.com/thingsboard/ngx-flowchart.git#a4157b0eef2eb3646ef920447c7b06b39d54f87f", + "version": "git://github.com/thingsboard/ngx-flowchart.git#97a77477ca8579becf0e3a07866046b4536fe30a", "from": "git://github.com/thingsboard/ngx-flowchart.git#master", "requires": { "tslib": "^1.10.0" diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 7e5ad6a8ec..8a25206c32 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -108,6 +108,7 @@ "@types/jstree": "^3.3.39", "@types/jszip": "^3.1.7", "@types/leaflet": "^1.5.12", + "@types/leaflet.markercluster": "^1.4.2", "@types/leaflet-polylinedecorator": "^1.6.0", "@types/lodash": "^4.14.150", "@types/raphael": "^2.3.0", diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index 39241c10d1..b0ab617a42 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -15,7 +15,7 @@ /// import _ from 'lodash'; -import { Observable, Subject, from, fromEvent, of } from 'rxjs'; +import { Observable, Subject, fromEvent, of } from 'rxjs'; import { finalize, share, map } from 'rxjs/operators'; import base64js from 'base64-js'; @@ -430,7 +430,7 @@ export function getDescendantProp(obj: any, path: string): any { export function imageLoader(imageUrl: string): Observable { const image = new Image(); - const imageLoad$ = fromEvent(image, 'load').pipe(map(event => image)); + const imageLoad$ = fromEvent(image, 'load').pipe(map(() => image)); image.src = imageUrl; return imageLoad$; } @@ -524,36 +524,49 @@ export function parseFunction(source: any, params: string[] = []): Function { return res; } -export function parseTemplate(template: string, data: object) { +export function parseTemplate(template: string, data: object, translateFn?: (key: string) => string) { let res = ''; + let variables = ''; try { - let variables = ''; - const expressions = template - .match(/\{(.*?)\}/g) // find expressions - .map(exp => exp.replace(/{|}/g, '')) // remove brackets - .map(exp => exp.split(':')) - .map(arr => { - variables += `let ${arr[0]} = ''; `; - return arr; - }) - .filter(arr => !!arr[1]) // filter expressions without format - .reduce((res, current) => { - res[current[0]] = current[1]; - return res; - }, {}); - - for (const key in data) { - if (!key.includes('|')) - variables += `${key} = '${expressions[key] ? padValue(data[key], +expressions[key]) : data[key]}';`; + if (template.match(//g, 'a>').replace(/name=(\'|")(.*?)(\'|")/g, `class='tb-custom-action' id='$2'`); + } + if (template.includes('i18n')) { + const translateRegexp = /\{i18n:(.*?)\}/; + template.match(new RegExp(translateRegexp.source, translateRegexp.flags + 'g')).forEach(match => { + template = template.replace(match, translateFn(match.match(translateRegexp)[1])); + }); + } + const expressions = template.match(/\{(.*?)\}/g); + if (expressions) { + const clearMatches = template.match(/(?<=\{)(.+?)(?=(\}|\:))/g); + for (const key in data) { + if (!key.includes('|')) + variables += `let ${key} = '${clearMatches[key] ? padValue(data[key], +clearMatches[key]) : data[key]}';`; + } + template = template.replace(/\:\d+\}/g, '}'); + res = safeExecute(parseFunction(variables + ' return' + '`' + template + '`')); } - template = template.replace(/:\d+}/g, '}'); - res = safeExecute(parseFunction(variables + 'return' + '`' + template + '`')); + else res = template; } catch (ex) { + console.log(ex, variables, template) } return res; } +export let parseWithTranslation = { + translate(): string { + throw console.error('Translate not assigned'); + }, + parseTemplate(template: string, data: object, forceTranslate = false): string { + return parseTemplate(forceTranslate ? this.translate(template) : template, data, this?.translate); + }, + setTranslate(translateFn: (key: string, defaultTranslation?: string) => string) { + this.translate = translateFn; + } +} + export function padValue(val: any, dec: number): string { let strVal; let n; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts index 90bd9341d6..da870813f3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts @@ -14,12 +14,12 @@ /// limitations under the License. /// -import L, { LatLngTuple } from 'leaflet'; +import L, { LatLngTuple, LatLngBounds, Point } from 'leaflet'; import 'leaflet-providers'; -import 'leaflet.markercluster/dist/MarkerCluster.css' -import 'leaflet.markercluster/dist/MarkerCluster.Default.css' -import 'leaflet.markercluster/dist/leaflet.markercluster' +import 'leaflet.markercluster/dist/MarkerCluster.css'; +import 'leaflet.markercluster/dist/MarkerCluster.Default.css'; +import LM from 'leaflet.markercluster/dist/leaflet.markercluster'; import { MapSettings, MarkerSettings, FormattedData, UnitedMapSettings, PolygonSettings, PolylineSettings } from './map-models'; import { Marker } from './markers'; @@ -34,7 +34,7 @@ export default abstract class LeafletMap { markers: Map = new Map(); polylines: Map = new Map(); polygons: Map = new Map(); - dragMode = true; + dragMode = false; map: L.Map; map$: BehaviorSubject = new BehaviorSubject(null); ready$: Observable = this.map$.pipe(filter(map => !!map)); @@ -43,6 +43,7 @@ export default abstract class LeafletMap { bounds: L.LatLngBounds; newMarker: L.Marker; datasources: FormattedData[]; + markersCluster: LM.markerClusterGroup; constructor(public $container: HTMLElement, options: UnitedMapSettings) { this.options = options; @@ -50,13 +51,38 @@ export default abstract class LeafletMap { public initSettings(options: MapSettings) { const { initCallback, - disableScrollZooming, }: MapSettings = options; + disableScrollZooming, + useClusterMarkers, + zoomOnClick, + showCoverageOnHover, + removeOutsideVisibleBounds, + animate, + chunkedLoading, + maxClusterRadius, + maxZoom }: MapSettings = options; if (disableScrollZooming) { this.map.scrollWheelZoom.disable(); } if (initCallback) { setTimeout(options.initCallback, 0); } + if (useClusterMarkers) { + const clusteringSettings: LM.MarkerClusterGroupOptions = { + zoomToBoundsOnClick: zoomOnClick, + showCoverageOnHover, + removeOutsideVisibleBounds, + animate, + chunkedLoading + }; + if (maxClusterRadius && maxClusterRadius > 0) { + clusteringSettings.maxClusterRadius = Math.floor(maxClusterRadius); + } + if (maxZoom && maxZoom >= 0 && maxZoom < 19) { + clusteringSettings.disableClusteringAtZoom = Math.floor(maxZoom); + } + this.markersCluster = LM.markerClusterGroup(clusteringSettings); + this.ready$.subscribe(map => map.addLayer(this.markersCluster)); + } } addMarkerControl() { @@ -65,7 +91,7 @@ export default abstract class LeafletMap { let addMarker: L.Control; this.map.on('mouseup', (e: L.LeafletMouseEvent) => { mousePositionOnMap = e.latlng; - }) + }); const dragListener = (e: L.DragEndEvent) => { if (e.type === 'dragend' && mousePositionOnMap) { const newMarker = L.marker(mousePositionOnMap).addTo(this.map); @@ -80,10 +106,17 @@ export default abstract class LeafletMap { this.saveMarkerLocation(updatedEnttity); this.map.removeLayer(newMarker); this.deleteMarker(ds.entityName); - this.createMarker(ds.entityName, updatedEnttity, this.datasources, this.options, false); + this.createMarker(ds.entityName, updatedEnttity, this.datasources, this.options); } datasourcesList.append(dsItem); - }) + }); + const deleteBtn = document.createElement('a'); + deleteBtn.appendChild(document.createTextNode('Delete position')); + deleteBtn.setAttribute('color', 'red'); + deleteBtn.onclick = () => { + this.map.removeLayer(newMarker); + } + datasourcesList.append(deleteBtn); const popup = L.popup(); popup.setContent(datasourcesList); newMarker.bindPopup(popup).openPopup(); @@ -96,6 +129,7 @@ export default abstract class LeafletMap { img.src = `assets/add_location.svg`; img.style.width = '32px'; img.style.height = '32px'; + img.title = 'Drag and drop to add marker'; img.onclick = this.dragMarker; img.draggable = true; const draggableImg = new L.Draggable(img); @@ -106,13 +140,10 @@ export default abstract class LeafletMap { onRemove(map) { }, dragMarker: this.dragMarker - } as any); - L.control.addMarker = (opts) => { return new L.Control.AddMarker(opts); } - addMarker = L.control.addMarker({ position: 'topright' }).addTo(this.map); } } @@ -164,9 +195,9 @@ export default abstract class LeafletMap { return this.map.getCenter(); } - fitBounds(bounds, useDefaultZoom = false) { + fitBounds(bounds: LatLngBounds, useDefaultZoom = false, padding?: LatLngTuple) { if (bounds.isValid()) { - if ((this.options.dontFitMapBounds || useDefaultZoom) && this.options.defaultZoomLevel) { + if ((!this.options.fitMapBounds || useDefaultZoom) && this.options.defaultZoomLevel) { this.map.setZoom(this.options.defaultZoomLevel, { animate: false }); this.map.panTo(bounds.getCenter(), { animate: false }); } else { @@ -175,7 +206,7 @@ export default abstract class LeafletMap { this.map.setZoom(this.options.minZoomLevel, { animate: false }); } }); - this.map.fitBounds(bounds, { padding: [50, 50], animate: false }); + this.map.fitBounds(bounds, { padding: padding || [50, 50], animate: false }); } this.bounds = this.bounds.extend(bounds); } @@ -193,8 +224,8 @@ export default abstract class LeafletMap { convertToCustomFormat(position: L.LatLng): object { return { - [this.options.latKeyName]: position.lat, - [this.options.lngKeyName]: position.lng + [this.options.latKeyName]: position.lat % 180, + [this.options.lngKeyName]: position.lng % 180 } } @@ -225,11 +256,17 @@ export default abstract class LeafletMap { this.saveMarkerLocation({ ...data, ...this.convertToCustomFormat(e.target._latlng) }); } - private createMarker(key: string, data: FormattedData, dataSources: FormattedData[], settings: MarkerSettings, setFocus = true) { + private createMarker(key: string, data: FormattedData, dataSources: FormattedData[], settings: MarkerSettings) { this.ready$.subscribe(() => { - const newMarker = new Marker(this.map, this.convertPosition(data), settings, data, dataSources, () => { }, this.dragMarker); - this.fitBounds(this.bounds.extend(newMarker.leafletMarker.getLatLng()), setFocus); + const newMarker = new Marker(this.convertPosition(data), settings, data, dataSources, this.dragMarker); + this.fitBounds(this.bounds.extend(newMarker.leafletMarker.getLatLng()), settings.draggableMarker && this.markers.size > 2); this.markers.set(key, newMarker); + if (this.options.useClusterMarkers) { + this.markersCluster.addLayer(newMarker.leafletMarker); + } + else { + this.map.addLayer(newMarker.leafletMarker); + } }); } @@ -242,6 +279,8 @@ export default abstract class LeafletMap { if (settings.showTooltip) { marker.updateMarkerTooltip(data); } + if (settings.useClusterMarkers) + this.markersCluster.refreshClusters() marker.setDataSources(data, dataSources); marker.updateMarkerIcon(settings); } @@ -320,7 +359,9 @@ export default abstract class LeafletMap { updatePolygon(key: string, data: LatLngTuple[], dataSources: DatasourceData[], settings: PolygonSettings) { this.ready$.subscribe(() => { - this.polygons.get(key).updatePolygon(data, dataSources, settings); + const poly = this.polygons.get(key); + poly.updatePolygon(data, dataSources, settings); + this.fitBounds(poly.leafletPoly.getBounds()); }); } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts index 1d2c993ef1..931f70f658 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { LatLngTuple } from 'leaflet'; +import { LatLngTuple, LeafletMouseEvent } from 'leaflet'; export type GenericFunction = (data: FormattedData, dsData: FormattedData[], dsIndex: number) => string; export type MarkerImageFunction = (data: FormattedData, dsData: FormattedData[], dsIndex: number) => string; @@ -24,14 +24,15 @@ export type MapSettings = { draggableMarker: boolean; initCallback?: () => any; defaultZoomLevel?: number; - dontFitMapBounds?: boolean; disableScrollZooming?: boolean; minZoomLevel?: number; + useClusterMarkers: boolean; latKeyName?: string; lngKeyName?: string; xPosKeyName?: string; yPosKeyName?: string; mapProvider: MapProviders; + mapProviderHere: string; mapUrl?: string; mapImageUrl?: string; provider?: MapProviders; @@ -42,6 +43,13 @@ export type MapSettings = { gmDefaultMapType?: string; useLabelFunction: string; icon?: any; + zoomOnClick: boolean, + maxZoom: number, + showCoverageOnHover: boolean, + animate: boolean, + maxClusterRadius: number, + chunkedLoading: boolean, + removeOutsideVisibleBounds: boolean } export enum MapProviders { @@ -54,6 +62,7 @@ export enum MapProviders { export type MarkerSettings = { tooltipPattern?: any; + tooltipAction: { [name: string]: actionsHandler }; icon?: any; showLabel?: boolean; label: string; @@ -62,19 +71,19 @@ export type MarkerSettings = { useLabelFunction: boolean; draggableMarker: boolean; showTooltip?: boolean; + useTooltipFunction: boolean; + useColorFunction: boolean; color?: string; autocloseTooltip: boolean; - displayTooltipAction: string; + showTooltipAction: string; + useClusterMarkers: boolean; currentImage?: string; useMarkerImageFunction?: boolean; markerImages?: string[]; - useMarkerImage: boolean; markerImageSize: number; fitMapBounds: boolean; - markerImage: { - length: number - } - + markerImage: string; + markerClick: { [name: string]: actionsHandler }; colorFunction: GenericFunction; tooltipFunction: GenericFunction; labelFunction: GenericFunction; @@ -98,16 +107,18 @@ export type PolygonSettings = { polygonStrokeColor: string; polygonColor: string; autocloseTooltip: boolean; - displayTooltipAction: string; - + showTooltipAction: string; + tooltipAction: object; + polygonClick: { [name: string]: actionsHandler }; polygonColorFunction?: GenericFunction; } export type PolylineSettings = { usePolylineDecorator: any; autocloseTooltip: boolean; - displayTooltipAction: string; + showTooltipAction: string; useColorFunction: any; + tooltipAction: { [name: string]: actionsHandler }; color: string; useStrokeOpacityFunction: any; strokeOpacity: number; @@ -131,4 +142,6 @@ export interface HistorySelectSettings { buttonColor: string; } -export type UnitedMapSettings = MapSettings & PolygonSettings & MarkerSettings & PolylineSettings; \ No newline at end of file +export type actionsHandler = ($event: Event | LeafletMouseEvent) => void; + +export type UnitedMapSettings = MapSettings & PolygonSettings & MarkerSettings & PolylineSettings; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.interface.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.interface.ts index f9a176137e..9900d98e0b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.interface.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.interface.ts @@ -15,6 +15,7 @@ /// import { JsonSettingsSchema } from '@shared/models/widget.models'; +import { MapProviders } from './map-models'; export interface MapWidgetInterface { resize(), @@ -24,8 +25,8 @@ export interface MapWidgetInterface { } export interface MapWidgetStaticInterface { - settingsSchema(mapProvider?, drawRoutes?): JsonSettingsSchema; - getProvidersSchema():JsonSettingsSchema + settingsSchema(mapProvider?: MapProviders, drawRoutes?: boolean): JsonSettingsSchema; + getProvidersSchema(mapProvider?: MapProviders): JsonSettingsSchema dataKeySettingsSchema(): object; actionSources(): object; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts index c2cd46c6b2..5794144bec 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts @@ -26,20 +26,23 @@ import { markerClusteringSettingsSchema, markerClusteringSettingsSchemaLeaflet, hereMapSettingsSchema, - mapProviderSchema + mapProviderSchema, + mapPolygonSchema } from './schemes'; import { MapWidgetStaticInterface, MapWidgetInterface } from './map-widget.interface'; import { OpenStreetMap, TencentMap, GoogleMap, HEREMap, ImageMap } from './providers'; -import { parseFunction, parseArray, parseData } from '@core/utils'; +import { parseFunction, parseArray, parseData, safeExecute, parseWithTranslation } from '@core/utils'; import { initSchema, addToSchema, mergeSchemes, addCondition, addGroupInfo } from '@core/schema-utils'; import { forkJoin } from 'rxjs'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; import { getDefCenterPosition } from './maps-utils'; -import { JsonSettingsSchema } from '@shared/models/widget.models'; +import { JsonSettingsSchema, WidgetActionDescriptor } from '@shared/models/widget.models'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { AttributeService } from '@core/http/attribute.service'; import { Type } from '@angular/core'; +import { TranslateService } from '@ngx-translate/core'; +import { UtilsService } from '@app/core/public-api'; // @dynamic export class MapWidgetController implements MapWidgetInterface { @@ -55,11 +58,16 @@ export class MapWidgetController implements MapWidgetInterface { $element = ctx.$container[0]; } this.settings = this.initSettings(ctx.settings); + this.settings.tooltipAction = this.getDescriptors('tooltipAction'); + this.settings.markerClick = this.getDescriptors('markerClick'); + this.settings.polygonClick = this.getDescriptors('polygonClick'); + // this.settings. const MapClass = providerSets[this.provider]?.MapClass; if (!MapClass) { return; } + parseWithTranslation.setTranslate(this.translate); this.map = new MapClass($element, this.settings); this.map.saveMarkerLocation = this.setMarkerLocation; } @@ -74,7 +82,8 @@ export class MapWidgetController implements MapWidgetInterface { return {}; } - public static getProvidersSchema() { + public static getProvidersSchema(mapProvider: MapProviders) { + mapProviderSchema.schema.properties.provider.default = mapProvider; return mergeSchemes([mapProviderSchema, ...Object.values(providerSets)?.map( (setting: IProvider) => addCondition(setting?.schema, `model.provider === '${setting.name}'`))]); @@ -82,19 +91,20 @@ export class MapWidgetController implements MapWidgetInterface { public static settingsSchema(mapProvider: MapProviders, drawRoutes: boolean): JsonSettingsSchema { const schema = initSchema(); - addToSchema(schema, this.getProvidersSchema()); - addGroupInfo(schema, 'Map Provider Settings'); - addToSchema(schema, commonMapSettingsSchema); + addToSchema(schema, this.getProvidersSchema(mapProvider)); + if(mapProvider!=='image-map'){ + addGroupInfo(schema, 'Map Provider Settings'); + addToSchema(schema, mergeSchemes([commonMapSettingsSchema, addCondition(mapPolygonSchema, 'model.showPolygon === true')])); addGroupInfo(schema, 'Common Map Settings'); - if (drawRoutes) { addToSchema(schema, routeMapSettingsSchema); addGroupInfo(schema, 'Route Map Settings'); } else if (mapProvider !== 'image-map') { - const clusteringSchema = mergeSchemes([markerClusteringSettingsSchemaLeaflet, markerClusteringSettingsSchema]) + const clusteringSchema = mergeSchemes([markerClusteringSettingsSchema, + addCondition(markerClusteringSettingsSchemaLeaflet, `model.useClusterMarkers === true`)]) addToSchema(schema, clusteringSchema); addGroupInfo(schema, 'Markers Clustering Settings'); - } + }} return schema; } @@ -115,9 +125,35 @@ export class MapWidgetController implements MapWidgetInterface { }; } + translate = (key: string, defaultTranslation?: string):string => { + return (this.ctx.$injector.get(UtilsService).customTranslation(key, defaultTranslation || key) + || this.ctx.$injector.get(TranslateService).instant(key)); + } + + getDescriptors(name: string): { [name: string]: ($event: Event) => void } { + const descriptors = this.ctx.actionsApi.getActionDescriptors(name); + const actions = {}; + descriptors.forEach(descriptor => { + actions[descriptor.name] = ($event: Event) => this.onCustomAction(descriptor, $event); + }, actions); + return actions; + } + onInit() { } + private onCustomAction(descriptor: WidgetActionDescriptor, $event: any) { + if ($event & $event.stopPropagation) { + $event?.stopPropagation(); + } + // safeExecute(parseFunction(descriptor.customFunction, ['$event', 'widgetContext']), [$event, this.ctx]) + const entityInfo = this.ctx.actionsApi.getActiveEntityInfo(); + const entityId = entityInfo ? entityInfo.entityId : null; + const entityName = entityInfo ? entityInfo.entityName : null; + const entityLabel = entityInfo ? entityInfo.entityLabel : null; + this.ctx.actionsApi.handleWidgetAction($event, descriptor, entityId, entityName, null, entityLabel); + } + setMarkerLocation = (e) => { const attributeService = this.ctx.$injector.get(AttributeService); forkJoin( @@ -136,13 +172,17 @@ export class MapWidgetController implements MapWidgetInterface { }] ); })).subscribe(res => { - console.log('MapWidgetController -> setMarkerLocation -> res', res) }); } initSettings(settings: UnitedMapSettings): UnitedMapSettings { const functionParams = ['data', 'dsData', 'dsIndex']; this.provider = settings.provider || this.mapProvider; + if (!settings.mapProviderHere) { + if (settings.mapProvider && hereProviders.includes(settings.mapProvider)) + settings.mapProviderHere = settings.mapProvider + else settings.mapProviderHere = hereProviders[0]; + } const customOptions = { provider: this.provider, mapUrl: settings?.mapImageUrl, @@ -156,7 +196,7 @@ export class MapWidgetController implements MapWidgetInterface { '${entityName}

Latitude: ${' + settings.latKeyName + ':7}
Longitude: ${' + settings.lngKeyName + ':7}', defaultCenterPosition: getDefCenterPosition(settings?.defaultCenterPosition), - currentImage: (settings.useMarkerImage && settings.markerImage?.length) ? { + currentImage: (settings.markerImage?.length) ? { url: settings.markerImage, size: settings.markerImageSize || 34 } : null @@ -194,7 +234,7 @@ interface IProvider { name: string } -export const providerSets: {[key: string]: IProvider} = { +export const providerSets: { [key: string]: IProvider } = { 'openstreet-map': { MapClass: OpenStreetMap, schema: openstreetMapSettingsSchema, @@ -236,7 +276,7 @@ export const defaultSettings: any = { useDefaultCenterPosition: false, showTooltipAction: 'click', autocloseTooltip: false, - showPolygon: true, + showPolygon: false, labelColor: '#000000', color: '#FE7569', polygonColor: '#0000ff', @@ -250,10 +290,16 @@ export const defaultSettings: any = { strokeOpacity: 1.0, initCallback: () => { }, defaultZoomLevel: 8, - dontFitMapBounds: false, disableScrollZooming: false, minZoomLevel: 16, credentials: '', markerClusteringSetting: null, - draggableMarker: false + draggableMarker: false, + fitMapBounds: true }; + +export const hereProviders = [ + 'HERE.normalDay', + 'HERE.normalNight', + 'HERE.hybridDay', + 'HERE.terrainDay'] diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts index 7af0828891..4ac437830c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts @@ -17,20 +17,32 @@ import L from 'leaflet'; import _ from 'lodash'; import { MarkerSettings, PolylineSettings, PolygonSettings } from './map-models'; +import { parseWithTranslation } from '@app/core/utils'; -export function createTooltip(target: L.Layer, settings: MarkerSettings | PolylineSettings | PolygonSettings): L.Popup { +export function createTooltip(target: L.Layer, + settings: MarkerSettings | PolylineSettings | PolygonSettings, + content?: string | HTMLElement): L.Popup { const popup = L.popup(); - popup.setContent(''); + popup.setContent(content); target.bindPopup(popup, { autoClose: settings.autocloseTooltip, closeOnClick: false }); - if (settings.displayTooltipAction === 'hover') { + if (settings.showTooltipAction === 'hover') { target.off('click'); target.on('mouseover', function () { - this.openPopup(); + target.openPopup(); }); target.on('mouseout', function () { - this.closePopup(); + target.closePopup(); }); } + target.on('popupopen', () => { + const actions = document.getElementsByClassName('tb-custom-action'); + Array.from(actions).forEach( + (element: HTMLElement) => { + if (element && settings.tooltipAction[element.id]) { + element.addEventListener('click', settings.tooltipAction[element.id]) + } + }); + }); return popup; } @@ -51,5 +63,4 @@ export function getDefCenterPosition(position) { if (typeof (position) === 'object') return position; return [0, 0]; -} - +} \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss index a017edeb9f..d402a4b5d9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss @@ -29,3 +29,7 @@ background: none; box-shadow: none; } + +.leaflet-container{ + background-color: white; +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts index 0903c5cd77..e032254b4d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import L from 'leaflet'; -import { MarkerSettings, FormattedData } from './map-models'; -import { aspectCache, safeExecute, parseTemplate } from '@app/core/utils'; +import { aspectCache, parseWithTranslation, safeExecute } from '@app/core/utils'; +import L, { LeafletMouseEvent } from 'leaflet'; +import { FormattedData, MarkerSettings } from './map-models'; import { createTooltip } from './maps-utils'; +import tinycolor from 'tinycolor2'; export class Marker { - leafletMarker: L.Marker; tooltipOffset: [number, number]; tooltip: L.Popup; @@ -28,8 +28,8 @@ export class Marker { data: FormattedData; dataSources: FormattedData[]; - constructor(private map: L.Map, location: L.LatLngExpression, public settings: MarkerSettings, - data?, dataSources?, onClickListener?, onDragendListener?) { + constructor(location: L.LatLngExpression, public settings: MarkerSettings, + data?: FormattedData, dataSources?, onDragendListener?) { this.setDataSources(data, dataSources); this.leafletMarker = L.marker(location, { draggable: settings.draggableMarker @@ -39,16 +39,21 @@ export class Marker { this.leafletMarker.setIcon(iconInfo.icon); this.tooltipOffset = [0, -iconInfo.size[1] + 10]; this.updateMarkerLabel(settings); - this.leafletMarker.addTo(map); }); if (settings.showTooltip) { this.tooltip = createTooltip(this.leafletMarker, settings); - this.tooltip.setContent(parseTemplate(this.settings.tooltipPattern, data)); + this.updateMarkerTooltip(data); } - if (onClickListener) { - this.leafletMarker.on('click', onClickListener); + if (this.settings.markerClick) { + this.leafletMarker.on('click', (event: LeafletMouseEvent) => { + for (const action in this.settings.markerClick) { + if (typeof (this.settings.markerClick[action]) === 'function') { + this.settings.markerClick[action](event); + } + } + }); } if (onDragendListener) { @@ -62,7 +67,9 @@ export class Marker { } updateMarkerTooltip(data: FormattedData) { - this.tooltip.setContent(parseTemplate(this.settings.tooltipPattern, data)); + const pattern = this.settings.useTooltipFunction ? + safeExecute(this.settings.tooltipFunction, [this.data, this.dataSources, this.data.dsIndex]) : this.settings.tooltipPattern; + this.tooltip.setContent(parseWithTranslation.parseTemplate(pattern, data, true)); } updateMarkerPosition(position: L.LatLngExpression) { @@ -71,13 +78,10 @@ export class Marker { updateMarkerLabel(settings: MarkerSettings) { this.leafletMarker.unbindTooltip(); - if (settings.showLabel) { - if (settings.useLabelFunction) { - settings.labelText = parseTemplate( - safeExecute(settings.labelFunction, [this.data, this.dataSources, this.data.dsIndex]), this.data) - } - else settings.labelText = parseTemplate(settings.label, this.data); + const pattern = settings.useLabelFunction ? + safeExecute(settings.labelFunction, [this.data, this.dataSources, this.data.dsIndex]) : settings.label; + settings.labelText = parseWithTranslation.parseTemplate(pattern, this.data, true); this.leafletMarker.bindTooltip(`
${settings.labelText}
`, { className: 'tb-marker-label', permanent: true, direction: 'top', offset: this.tooltipOffset }); } @@ -98,7 +102,6 @@ export class Marker { } createMarkerIcon(onMarkerIconReady) { - if (this.settings.icon) { onMarkerIconReady({ size: [30, 30], @@ -106,11 +109,11 @@ export class Marker { }); return; } - const currentImage = this.settings.useMarkerImageFunction ? safeExecute(this.settings.markerImageFunction, [this.data, this.settings.markerImages, this.dataSources, this.data.dsIndex]) : this.settings.currentImage; - + const currentColor = tinycolor(this.settings.useColorFunction ? safeExecute(this.settings.colorFunction, + [this.data, this.dataSources, this.data.dsIndex]) : this.settings.color).toHex(); if (currentImage && currentImage.url) { aspectCache(currentImage.url).subscribe( @@ -137,19 +140,19 @@ export class Marker { }; onMarkerIconReady(iconInfo); } else { - this.createDefaultMarkerIcon(this.settings.color, onMarkerIconReady); + this.createDefaultMarkerIcon(currentColor, onMarkerIconReady); } } ); } else { - this.createDefaultMarkerIcon(this.settings.color, onMarkerIconReady); + this.createDefaultMarkerIcon(currentColor, onMarkerIconReady); } } createDefaultMarkerIcon(color, onMarkerIconReady) { const pinColor = color.substr(1); const icon = L.icon({ - iconUrl: 'https://chart.apis.google.com/chart?chst=d_map_pin_letter&chld=%E2%80%A2|' + pinColor, + iconUrl: 'https://chart.apis.google.com/chart?chst=d_map_pin_letter&chld=%E2%80%A2|' + color, iconSize: [21, 34], iconAnchor: [10, 34], popupAnchor: [0, -34], diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts index 60221b62a3..2b48e3ee8b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts @@ -20,14 +20,9 @@ import { MapSettings, UnitedMapSettings } from '../map-models'; export class HEREMap extends LeafletMap { constructor($container, options: UnitedMapSettings) { - const defaultCredentials = - { - app_id: 'AhM6TzD9ThyK78CT3ptx', - app_code: 'p6NPiITB3Vv0GMUFnkLOOg' - } super($container, options); const map = L.map($container).setView(options?.defaultCenterPosition, options?.defaultZoomLevel); - const tileLayer = (L.tileLayer as any).provider(options.mapProvider || 'HERE.normalDay', options.credentials || defaultCredentials); + const tileLayer = (L.tileLayer as any).provider(options.mapProviderHere || 'HERE.normalDay', options.credentials); tileLayer.addTo(map); super.setMap(map); super.initSettings(options); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts index 4222ad11fd..64e3a39543 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts @@ -110,7 +110,7 @@ export const hereMapSettingsSchema = title: 'HERE Map Configuration', type: 'object', properties: { - mapProvider: { + mapProviderHere: { title: 'Map layer', type: 'string', default: 'HERE.normalDay' @@ -120,11 +120,13 @@ export const hereMapSettingsSchema = properties: { app_id: { title: 'HERE app id', - type: 'string' + type: 'string', + default: 'AhM6TzD9ThyK78CT3ptx' }, app_code: { title: 'HERE app code', - type: 'string' + type: 'string', + default: 'p6NPiITB3Vv0GMUFnkLOOg' } }, required: ['app_id', 'app_code'] @@ -134,7 +136,7 @@ export const hereMapSettingsSchema = }, form: [ { - key: 'mapProvider', + key: 'mapProviderHere', type: 'rc-select', multiple: false, items: [ @@ -339,43 +341,6 @@ export const commonMapSettingsSchema = type: 'boolean', default: false }, - polygonKeyName: { - title: 'Polygon key name', - type: 'string', - default: 'coordinates' - }, - polygonColor: { - title: 'Polygon color', - type: 'string' - }, - polygonOpacity: { - title: 'Polygon opacity', - type: 'number', - default: 0.5 - }, - polygonStrokeColor: { - title: 'Stroke color', - type: 'string' - }, - polygonStrokeOpacity: { - title: 'Stroke opacity', - type: 'number', - default: 1 - }, - polygonStrokeWeight: { - title: 'Stroke weight', - type: 'number', - default: 1 - }, - usePolygonColorFunction: { - title: 'Use polygon color function', - type: 'boolean', - default: false - }, - polygonColorFunction: { - title: 'Polygon Color function: f(data, dsData, dsIndex)', - type: 'string' - }, markerImage: { title: 'Custom marker image', type: 'string' @@ -455,20 +420,6 @@ export const commonMapSettingsSchema = { key: 'colorFunction', type: 'javascript' - }, 'showPolygon', 'polygonKeyName', - { - key: 'polygonColor', - type: 'color' - }, - 'polygonOpacity', - { - key: 'polygonStrokeColor', - type: 'color' - }, - 'polygonStrokeOpacity', 'polygonStrokeWeight', 'usePolygonColorFunction', - { - key: 'polygonColorFunction', - type: 'javascript' }, { key: 'markerImage', @@ -488,7 +439,73 @@ export const commonMapSettingsSchema = type: 'image' } ] - } + }, + 'showPolygon', + ] +}; + +export const mapPolygonSchema = +{ + schema: { + title: 'Map Polygon Configuration', + type: 'object', + properties: { + polygonKeyName: { + title: 'Polygon key name', + type: 'string', + default: 'coordinates' + }, + polygonColor: { + title: 'Polygon color', + type: 'string' + }, + polygonOpacity: { + title: 'Polygon opacity', + type: 'number', + default: 0.5 + }, + polygonStrokeColor: { + title: 'Stroke color', + type: 'string' + }, + polygonStrokeOpacity: { + title: 'Stroke opacity', + type: 'number', + default: 1 + }, + polygonStrokeWeight: { + title: 'Stroke weight', + type: 'number', + default: 1 + }, + usePolygonColorFunction: { + title: 'Use polygon color function', + type: 'boolean', + default: false + }, + polygonColorFunction: { + title: 'Polygon Color function: f(data, dsData, dsIndex)', + type: 'string' + }, + }, + required: [] + }, + form: [ + 'polygonKeyName', + { + key: 'polygonColor', + type: 'color' + }, + 'polygonOpacity', + { + key: 'polygonStrokeColor', + type: 'color' + }, + 'polygonStrokeOpacity', 'polygonStrokeWeight', 'usePolygonColorFunction', + { + key: 'polygonColorFunction', + type: 'javascript' + }, ] }; @@ -527,23 +544,12 @@ export const markerClusteringSettingsSchema = title: 'Use map markers clustering', type: 'boolean', default: false - }, - zoomOnClick: { - title: 'Zoom when clicking on a cluster', - type: 'boolean', - default: true - }, - maxZoom: { - title: 'The maximum zoom level when a marker can be part of a cluster (0 - 18)', - type: 'number' } }, required: [] }, form: [ 'useClusterMarkers', - 'zoomOnClick', - 'maxZoom' ] }; @@ -571,12 +577,23 @@ export const markerClusteringSettingsSchemaGoogle = ] }; + + export const markerClusteringSettingsSchemaLeaflet = { schema: { title: 'Markers Clustering Configuration Leaflet', type: 'object', properties: { + zoomOnClick: { + title: 'Zoom when clicking on a cluster', + type: 'boolean', + default: true + }, + maxZoom: { + title: 'The maximum zoom level when a marker can be part of a cluster (0 - 18)', + type: 'number' + }, showCoverageOnHover: { title: 'Show the bounds of markers when mouse over a cluster', type: 'boolean', @@ -606,6 +623,8 @@ export const markerClusteringSettingsSchemaLeaflet = required: [] }, form: [ + 'zoomOnClick', + 'maxZoom', 'showCoverageOnHover', 'animate', 'maxClusterRadius', diff --git a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.html b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.html index 1b21fe253f..2b070052a5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.html @@ -17,17 +17,17 @@ -->
- {{settings.label | tbParseTemplate: activeTrip}} + {{label }}
-
+
-
+
-
diff --git a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.scss b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.scss index d7be4629c3..cdb28f8a27 100644 --- a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.scss @@ -74,10 +74,10 @@ .trip-animation-tooltip { position: absolute; - top: 38px; + top: 30px; right: 0; - z-index: 400; - padding: 10px; + z-index: 1000; + padding: 5px; background-color: #fff; transition: 0.3s ease-in-out; @@ -86,4 +86,4 @@ } } } -} +} \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts index 21e3f87873..5d2f8bfa2e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts @@ -22,13 +22,14 @@ import { interpolateOnPointSegment } from 'leaflet-geometryutil'; import { Component, OnInit, Input, ViewChild, AfterViewInit, ChangeDetectorRef, SecurityContext } from '@angular/core'; import { MapWidgetController, TbMapWidgetV2 } from '../lib/maps/map-widget2'; import { MapProviders } from '../lib/maps/map-models'; -import { parseArray, parseTemplate, safeExecute } from '@app/core/utils'; +import { parseArray, parseWithTranslation, safeExecute, parseTemplate } from '@app/core/utils'; import { initSchema, addToSchema, addGroupInfo } from '@app/core/schema-utils'; import { tripAnimationSchema } from '../lib/maps/schemes'; import { DomSanitizer } from '@angular/platform-browser'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; import { getRatio, findAngle } from '../lib/maps/maps-utils'; import { JsonSettingsSchema, WidgetConfig } from '@shared/models/widget.models'; +import moment from 'moment'; @Component({ @@ -55,6 +56,9 @@ export class TripAnimationComponent implements OnInit, AfterViewInit { mainTooltip = ''; visibleTooltip = false; activeTrip; + label; + minTime; + maxTime; static getSettingsSchema(): JsonSettingsSchema { const schema = initSchema(); @@ -90,48 +94,58 @@ export class TripAnimationComponent implements OnInit, AfterViewInit { ngAfterViewInit() { const ctxCopy: WidgetContext = _.cloneDeep(this.ctx); - ctxCopy.settings.showLabel = false; - ctxCopy.settings.showTooltip = false; this.mapWidget = new MapWidgetController(MapProviders.openstreet, false, ctxCopy, this.mapContainer.nativeElement); } timeUpdated(time: number) { const currentPosition = this.interpolatedData.map(dataSource => dataSource[time]); this.activeTrip = currentPosition[0]; - if (this.settings.showPolygon) { - this.mapWidget.map.updatePolygons(this.interpolatedData); + this.minTime = moment(this.historicalData[0][this.historicalData.length - 1]?.time).format('YYYY-MM-DD HH:mm:ss') + this.maxTime = moment(this.historicalData[0][0]?.time).format('YYYY-MM-DD HH:mm:ss') + this.calcLabel(); + this.calcTooltip(); + if (this.mapWidget) { + if (this.settings.showPolygon) { + this.mapWidget.map.updatePolygons(this.interpolatedData); + } + this.mapWidget.map.updateMarkers(currentPosition); } - this.mapWidget.map.updateMarkers(currentPosition); } setActiveTrip() { - } calculateIntervals() { this.historicalData.forEach((dataSource, index) => { this.intervals = []; + for (let time = dataSource[0]?.time; time < dataSource[dataSource.length - 1]?.time; time += this.normalizationStep) { this.intervals.push(time); } + this.intervals.push(dataSource[dataSource.length - 1]?.time); this.interpolatedData[index] = this.interpolateArray(dataSource, this.intervals); }); + } - showHideTooltip() { + calcTooltip() { + const data = { ...this.activeTrip, maxTime: this.maxTime, minTime: this.minTime } const tooltipText: string = this.settings.useTooltipFunction ? - safeExecute(this.settings.tooolTipFunction, [this.activeTrip, this.historicalData, 0]) - : this.settings.tooltipPattern; + safeExecute(this.settings.tooolTipFunction, [data, this.historicalData, 0]) : this.settings.tooltipPattern; + this.mainTooltip = this.sanitizer.sanitize( + SecurityContext.HTML, (parseWithTranslation.parseTemplate(tooltipText, data, true))); + } - this.mainTooltip = this.sanitizer.sanitize(SecurityContext.HTML, parseTemplate(tooltipText, this.activeTrip)) - this.visibleTooltip = !this.visibleTooltip; + calcLabel() { + const data = { ...this.activeTrip, maxTime: this.maxTime, minTime: this.minTime } + const labelText: string = this.settings.useLabelFunction ? + safeExecute(this.settings.labelFunction, [data, this.historicalData, 0]) : this.settings.label; + this.label = (parseWithTranslation.parseTemplate(labelText, data, true)); } interpolateArray(originData, interpolatedIntervals) { - const result = {}; - for (let i = 1, j = 0; i < originData.length && j < interpolatedIntervals.length;) { const currentTime = interpolatedIntervals[j]; while (originData[i].time < currentTime) i++; diff --git a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.html b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.html index 05d6b20c89..54bba32bae 100644 --- a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.html +++ b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.html @@ -16,15 +16,21 @@ -->
-
+
- - +
+ + +
+ {{ this.intervals[this.index] | date:'medium'}} + {{ "widget.no-data-found" | translate}} +
+
@@ -41,11 +47,8 @@ pause_circle_outline - - {{speedValue}} + + {{speedValue}} -
-
- {{ this.intervals[this.index] | date:'medium'}} - {{ "widget.no-data-found" | translate}} -
\ No newline at end of file +
\ No newline at end of file diff --git a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss index d4e4ae17ed..0ca2a9e5b8 100644 --- a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss +++ b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss @@ -13,128 +13,120 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -.trip-animation-widget { + +.trip-animation-label-container { + height: 24px; +} + +.trip-animation-container { position: relative; + z-index: 1; + flex: 1; width: 100%; - height: 100%; - font-size: 16px; - line-height: 24px; - .trip-animation-label-container { - height: 24px; - } - - .trip-animation-container { - position: relative; + #trip-animation-map { z-index: 1; - flex: 1; width: 100%; + height: 100%; - #trip-animation-map { - z-index: 1; - width: 100%; - height: 100%; - - .pointsLayerMarkerIcon { - border-radius: 50%; - } + .pointsLayerMarkerIcon { + border-radius: 50%; } + } + + .trip-animation-info-panel { + position: absolute; + top: 0; + right: 0; + z-index: 2; + pointer-events: none; - .trip-animation-info-panel { - position: absolute; + .mat-button { top: 0; - right: 0; - z-index: 2; - pointer-events: none; - - .md-button { - top: 0; - left: 0; - width: 32px; - min-width: 32px; - height: 32px; - min-height: 32px; - padding: 0 0 2px; - margin: 2px; - line-height: 24px; - - ng-md-icon { - width: 24px; - height: 24px; - - svg { - width: inherit; - height: inherit; - } + left: 0; + width: 32px; + min-width: 32px; + height: 32px; + min-height: 32px; + padding: 0 0 2px; + margin: 2px; + line-height: 24px; + + ng-mat-icon { + width: 24px; + height: 24px; + + svg { + width: inherit; + height: inherit; } } } + } - .trip-animation-tooltip { - position: absolute; - top: 38px; - right: 0; - z-index: 2; - padding: 10px; - background-color: #fff; - transition: 0.3s ease-in-out; - - &-hidden { - transform: translateX(110%); - } + .trip-animation-tooltip { + position: absolute; + top: 38px; + right: 0; + z-index: 2; + padding: 10px; + background-color: #fff; + transition: 0.3s ease-in-out; + + &-hidden { + transform: translateX(110%); } } +} - .trip-animation-control-panel { - position: relative; - box-sizing: border-box; - width: 100%; - padding-bottom: 16px; - padding-left: 10px; +.trip-animation-control-panel { + position: relative; + box-sizing: border-box; + width: 100%; + padding-bottom: 16px; + padding-left: 10px; - md-slider-container { - md-slider { - min-width: 80px; - } + mat-slider-container { + mat-slider { + min-width: 80px; + } - button.md-button.md-icon-button { - width: 44px; - min-width: 44px; - height: 44px; - min-height: 44px; - margin: 0; - line-height: 28px; - - md-icon { - width: 28px; - height: 28px; - font-size: 28px; - - svg { - width: inherit; - height: inherit; - } + button.mat-button.mat-icon-button { + width: 44px; + min-width: 44px; + height: 44px; + min-height: 44px; + margin: 0; + line-height: 28px; + + mat-icon { + width: 28px; + height: 28px; + font-size: 28px; + + svg { + width: inherit; + height: inherit; } } - - md-select { - margin: 0; - } } - .panel-timer { - max-width: none; - padding-right: 250px; - padding-left: 90px; - margin-top: -20px; - font-size: 12px; - font-weight: 500; - text-align: center; + mat-select { + margin: 0; } } + + .panel-timer { + max-width: none; + margin-top: -20px; + font-size: 12px; + font-weight: 500; + text-align: center; + } } -.speed-select{ +.speed-select { width: 50px; - margin-left: 20px; + margin-left: 10px; + margin-top: 10px; } diff --git a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts index f8659de2d5..252720033c 100644 --- a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts +++ b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts @@ -15,8 +15,8 @@ /// import { Component, OnInit, OnChanges, Input, Output, EventEmitter, ChangeDetectorRef } from '@angular/core'; -import { interval, Subscription, Subscriber, SubscriptionLike, Observer } from 'rxjs'; -import { filter, tap } from 'rxjs/operators'; +import { interval } from 'rxjs'; +import { filter } from 'rxjs/operators'; import { HistorySelectSettings } from '@app/modules/home/components/widget/lib/maps/map-models'; @Component({ diff --git a/ui-ngx/src/app/shared/pipe/template.pipe.ts b/ui-ngx/src/app/shared/pipe/template.pipe.ts index 4970672325..2cce658adb 100644 --- a/ui-ngx/src/app/shared/pipe/template.pipe.ts +++ b/ui-ngx/src/app/shared/pipe/template.pipe.ts @@ -15,11 +15,11 @@ /// import { Pipe, PipeTransform } from '@angular/core'; -import { parseTemplate } from '@app/core/utils'; +import { parseTemplate, parseWithTranslation } from '@app/core/utils'; @Pipe({ name: 'tbParseTemplate' }) export class TbTemplatePipe implements PipeTransform { transform(template, data): string { - return parseTemplate(template, data); + return parseWithTranslation.parseTemplate(template, data); } } From 9c5ad303cdce84e3841d46dc2d94d8a3dda2c9b5 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Fri, 24 Apr 2020 15:49:53 +0300 Subject: [PATCH 089/102] Add support Safari and improvement view (#2662) --- .../entity/add-entity-dialog.component.ts | 1 - .../dashboard/dashboard-page.component.html | 64 +++++++++---------- .../default-state-controller.component.scss | 2 + .../entity-state-controller.component.html | 2 +- .../entity-state-controller.component.scss | 6 +- ...age-dashboard-states-dialog.component.html | 8 +-- ...anage-dashboard-states-dialog.component.ts | 11 +--- .../save-widget-type-as-dialog.component.html | 11 ++-- .../select-widget-type-dialog.component.html | 10 ++- .../select-widget-type-dialog.component.scss | 3 +- .../pages/widget/widget-editor.component.html | 10 ++- .../pages/widget/widget-editor.component.scss | 30 +++++---- .../dashboard-select-panel.component.scss | 4 -- .../dashboard-select.component.scss | 1 + 14 files changed, 80 insertions(+), 83 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts index 9793c1de92..2a39250872 100644 --- a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts @@ -29,7 +29,6 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod import { AddEntityDialogData } from '@home/models/entity/entity-component.models'; import { DialogComponent } from '@shared/components/dialog.component'; import { Router } from '@angular/router'; -import { Observable } from 'rxjs'; @Component({ selector: 'tb-add-entity-dialog', diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html index 8c85c92606..76a59242a9 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html @@ -23,9 +23,39 @@ 'tb-dashboard-toolbar-closed': !toolbarOpened }"> -
-
+ + +
+ + +
+ + +
+
-
- - -
- - -
- - -
diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/default-state-controller.component.scss b/ui-ngx/src/app/modules/home/pages/dashboard/states/default-state-controller.component.scss index 091a7132dd..919fbbf16f 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/default-state-controller.component.scss +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/default-state-controller.component.scss @@ -14,6 +14,8 @@ * limitations under the License. */ :host { + width: min-content; //for Safari + mat-select.default-state-controller { margin: 0; } diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.html index 7d32c49051..5d03bcd107 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.scss b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.scss index ffe6591fea..6aa314c46d 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.scss +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.scss @@ -14,12 +14,12 @@ * limitations under the License. */ :host { + overflow: hidden; + .entity-state-controller { .state-divider { - padding-right: 15px; - padding-left: 15px; + padding: 0 15px; overflow: hidden; - font-size: 18px; text-overflow: ellipsis; white-space: nowrap; pointer-events: none; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.html index c4d0fb61dc..857f0c2565 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.html @@ -28,8 +28,8 @@
-
-
+
+
@@ -78,13 +78,13 @@ - {{ 'dashboard.state-name' | translate }} + {{ 'dashboard.state-name' | translate }} {{ state.name }} - {{ 'dashboard.state-id' | translate }} + {{ 'dashboard.state-id' | translate }} {{ state.id }} diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.ts index 40d9cd0d6f..ad32146f87 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/manage-dashboard-states-dialog.component.ts @@ -16,19 +16,14 @@ import { AfterViewInit, Component, ElementRef, Inject, OnInit, SkipSelf, ViewChild } from '@angular/core'; import { ErrorStateMatcher } from '@angular/material/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@app/shared/components/dialog.component'; import { DashboardState } from '@app/shared/models/dashboard.models'; -import { MatDialog } from '@angular/material/dialog'; import { PageLink } from '@shared/models/page/page-link'; -import { - WidgetActionDescriptorInfo, - WidgetActionsDatasource -} from '@home/components/widget/action/manage-widget-actions.component.models'; import { DashboardStateInfo, DashboardStatesDatasource @@ -40,10 +35,6 @@ import { fromEvent, merge } from 'rxjs'; import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators'; import { TranslateService } from '@ngx-translate/core'; import { DialogService } from '@core/services/dialog.service'; -import { - WidgetActionDialogComponent, - WidgetActionDialogData -} from '@home/components/widget/action/widget-action-dialog.component'; import { deepClone } from '@core/utils'; import { DashboardStateDialogComponent, diff --git a/ui-ngx/src/app/modules/home/pages/widget/save-widget-type-as-dialog.component.html b/ui-ngx/src/app/modules/home/pages/widget/save-widget-type-as-dialog.component.html index 1bb2f1a3ec..c611d82bc1 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/save-widget-type-as-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/widget/save-widget-type-as-dialog.component.html @@ -15,9 +15,8 @@ limitations under the License. --> -
- + +

widget.save-widget-type-as

+ \n \n\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),I=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.assignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.assignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],createCustomerIfNotExists:[!!e&&e.createCustomerIfNotExists,[]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n {{ \'tb.rulenode.create-customer-if-not-exists\' | translate }}\n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),k=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.clearAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.clearAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],alarmType:[e?e.alarmType:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.clearAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.clearAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-clear-alarm-config",template:'
\n \n \n \n
\n \n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),N=function(e){function r(t,r,n,o){var i=e.call(this,t)||this;return i.store=t,i.fb=r,i.nodeScriptTestService=n,i.translate=o,i.alarmSeverities=Object.keys(a.AlarmSeverity),i.alarmSeverityTranslationMap=a.alarmSeverityTranslations,i.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],i}return y(r,e),r.prototype.configForm=function(){return this.createAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.createAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],useMessageAlarmData:[!!e&&e.useMessageAlarmData,[]],alarmType:[e?e.alarmType:null,[]],severity:[e?e.severity:null,[]],propagate:[!!e&&e.propagate,[]],relationTypes:[e?e.relationTypes:null,[]]})},r.prototype.validatorTriggers=function(){return["useMessageAlarmData"]},r.prototype.updateValidators=function(e){this.createAlarmConfigForm.get("useMessageAlarmData").value?(this.createAlarmConfigForm.get("alarmType").setValidators([]),this.createAlarmConfigForm.get("severity").setValidators([])):(this.createAlarmConfigForm.get("alarmType").setValidators([i.Validators.required]),this.createAlarmConfigForm.get("severity").setValidators([i.Validators.required])),this.createAlarmConfigForm.get("alarmType").updateValueAndValidity({emitEvent:e}),this.createAlarmConfigForm.get("severity").updateValueAndValidity({emitEvent:e})},r.prototype.testScript=function(){var e=this,t=this.createAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.createAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.removeKey=function(e,t){var r=this.createAlarmConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.createAlarmConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.createAlarmConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.createAlarmConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-create-alarm-config",template:'
\n \n \n \n
\n \n
\n \n {{ \'tb.rulenode.use-message-alarm-data\' | translate }}\n \n
\n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n \n tb.rulenode.alarm-severity\n \n \n {{ alarmSeverityTranslationMap.get(severity) | translate }}\n \n \n \n {{ \'tb.rulenode.alarm-severity-required\' | translate }}\n \n \n
\n \n {{ \'tb.rulenode.propagate\' | translate }}\n \n
\n \n tb.rulenode.relation-types-list\n \n \n {{key}}\n close\n \n \n \n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),V=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.createRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.createRelationConfigForm=this.fb.group({direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[i.Validators.required]],entityNamePattern:[e?e.entityNamePattern:null,[]],entityTypePattern:[e?e.entityTypePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],createEntityIfNotExists:[!!e&&e.createEntityIfNotExists,[]],removeCurrentRelations:[!!e&&e.removeCurrentRelations,[]],changeOriginatorToRelatedEntity:[!!e&&e.changeOriginatorToRelatedEntity,[]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["entityType"]},r.prototype.updateValidators=function(e){var t=this.createRelationConfigForm.get("entityType").value;t?this.createRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.createRelationConfigForm.get("entityNamePattern").setValidators([]),!t||t!==a.EntityType.DEVICE&&t!==a.EntityType.ASSET?this.createRelationConfigForm.get("entityTypePattern").setValidators([]):this.createRelationConfigForm.get("entityTypePattern").setValidators([i.Validators.required]),this.createRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e}),this.createRelationConfigForm.get("entityTypePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-create-relation-config",template:'
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-type-pattern\n \n \n {{ \'tb.rulenode.entity-type-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n
\n \n {{ \'tb.rulenode.create-entity-if-not-exists\' | translate }}\n \n
tb.rulenode.create-entity-if-not-exists-hint
\n
\n \n {{ \'tb.rulenode.remove-current-relations\' | translate }}\n \n
tb.rulenode.remove-current-relations-hint
\n \n {{ \'tb.rulenode.change-originator-to-related-entity\' | translate }}\n \n
tb.rulenode.change-originator-to-related-entity-hint
\n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),E=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgDelayConfigForm},r.prototype.onConfigurationSet=function(e){this.msgDelayConfigForm=this.fb.group({useMetadataPeriodInSecondsPatterns:[!!e&&e.useMetadataPeriodInSecondsPatterns,[]],periodInSeconds:[e?e.periodInSeconds:null,[]],periodInSecondsPattern:[e?e.periodInSecondsPattern:null,[]],maxPendingMsgs:[e?e.maxPendingMsgs:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(1e5)]]})},r.prototype.validatorTriggers=function(){return["useMetadataPeriodInSecondsPatterns"]},r.prototype.updateValidators=function(e){this.msgDelayConfigForm.get("useMetadataPeriodInSecondsPatterns").value?(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([i.Validators.required]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([])):(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([i.Validators.required,i.Validators.min(0)])),this.msgDelayConfigForm.get("periodInSecondsPattern").updateValueAndValidity({emitEvent:e}),this.msgDelayConfigForm.get("periodInSeconds").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-delay-config",template:'
\n \n {{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
\n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-0-seconds-message\' | translate }}\n \n \n \n \n tb.rulenode.period-in-seconds-pattern\n \n \n {{ \'tb.rulenode.period-in-seconds-pattern-required\' | translate }}\n \n \n \n \n \n tb.rulenode.max-pending-messages\n \n \n {{ \'tb.rulenode.max-pending-messages-required\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),A=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.deleteRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.deleteRelationConfigForm=this.fb.group({deleteForSingleEntity:[!!e&&e.deleteForSingleEntity,[]],direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[]],entityNamePattern:[e?e.entityNamePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["deleteForSingleEntity","entityType"]},r.prototype.updateValidators=function(e){var t=this.deleteRelationConfigForm.get("deleteForSingleEntity").value,r=this.deleteRelationConfigForm.get("entityType").value;t?this.deleteRelationConfigForm.get("entityType").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityType").setValidators([]),t&&r?this.deleteRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityNamePattern").setValidators([]),this.deleteRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:!1}),this.deleteRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-delete-relation-config",template:'
\n \n {{ \'tb.rulenode.delete-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.delete-relation-hint
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),L=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.generatorConfigForm},r.prototype.onConfigurationSet=function(e){this.generatorConfigForm=this.fb.group({msgCount:[e?e.msgCount:null,[i.Validators.required,i.Validators.min(0)]],periodInSeconds:[e?e.periodInSeconds:null,[i.Validators.required,i.Validators.min(1)]],originator:[e?e.originator:null,[]],jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.prepareInputConfig=function(e){return e&&(e.originatorId&&e.originatorType?e.originator={id:e.originatorId,entityType:e.originatorType}:e.originator=null,delete e.originatorId,delete e.originatorType),e},r.prototype.prepareOutputConfig=function(e){return e.originator?(e.originatorId=e.originator.id,e.originatorType=e.originator.entityType):(e.originatorId=null,e.originatorType=null),delete e.originator,e},r.prototype.testScript=function(){var e=this,t=this.generatorConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"generate",this.translate.instant("tb.rulenode.generator"),"Generate",["prevMsg","prevMetadata","prevMsgType"],this.ruleNodeId).subscribe((function(t){t&&e.generatorConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-generator-config",template:'
\n \n tb.rulenode.message-count\n \n \n {{ \'tb.rulenode.message-count-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-message-count-message\' | translate }}\n \n \n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-seconds-message\' | translate }}\n \n \n
\n \n \n \n
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent);!function(e){e.CUSTOMER="CUSTOMER",e.TENANT="TENANT",e.RELATED="RELATED",e.ALARM_ORIGINATOR="ALARM_ORIGINATOR"}(v||(v={}));var M,P=new Map([[v.CUSTOMER,"tb.rulenode.originator-customer"],[v.TENANT,"tb.rulenode.originator-tenant"],[v.RELATED,"tb.rulenode.originator-related"],[v.ALARM_ORIGINATOR,"tb.rulenode.originator-alarm-originator"]]);!function(e){e.CIRCLE="CIRCLE",e.POLYGON="POLYGON"}(M||(M={}));var R,w=new Map([[M.CIRCLE,"tb.rulenode.perimeter-circle"],[M.POLYGON,"tb.rulenode.perimeter-polygon"]]);!function(e){e.MILLISECONDS="MILLISECONDS",e.SECONDS="SECONDS",e.MINUTES="MINUTES",e.HOURS="HOURS",e.DAYS="DAYS"}(R||(R={}));var D,O=new Map([[R.MILLISECONDS,"tb.rulenode.time-unit-milliseconds"],[R.SECONDS,"tb.rulenode.time-unit-seconds"],[R.MINUTES,"tb.rulenode.time-unit-minutes"],[R.HOURS,"tb.rulenode.time-unit-hours"],[R.DAYS,"tb.rulenode.time-unit-days"]]);!function(e){e.METER="METER",e.KILOMETER="KILOMETER",e.FOOT="FOOT",e.MILE="MILE",e.NAUTICAL_MILE="NAUTICAL_MILE"}(D||(D={}));var K,B=new Map([[D.METER,"tb.rulenode.range-unit-meter"],[D.KILOMETER,"tb.rulenode.range-unit-kilometer"],[D.FOOT,"tb.rulenode.range-unit-foot"],[D.MILE,"tb.rulenode.range-unit-mile"],[D.NAUTICAL_MILE,"tb.rulenode.range-unit-nautical-mile"]]);!function(e){e.TITLE="TITLE",e.COUNTRY="COUNTRY",e.STATE="STATE",e.ZIP="ZIP",e.ADDRESS="ADDRESS",e.ADDRESS2="ADDRESS2",e.PHONE="PHONE",e.EMAIL="EMAIL",e.ADDITIONAL_INFO="ADDITIONAL_INFO"}(K||(K={}));var j,U,G,H=new Map([[K.TITLE,"tb.rulenode.entity-details-title"],[K.COUNTRY,"tb.rulenode.entity-details-country"],[K.STATE,"tb.rulenode.entity-details-state"],[K.ZIP,"tb.rulenode.entity-details-zip"],[K.ADDRESS,"tb.rulenode.entity-details-address"],[K.ADDRESS2,"tb.rulenode.entity-details-address2"],[K.PHONE,"tb.rulenode.entity-details-phone"],[K.EMAIL,"tb.rulenode.entity-details-email"],[K.ADDITIONAL_INFO,"tb.rulenode.entity-details-additional_info"]]);!function(e){e.FIRST="FIRST",e.LAST="LAST",e.ALL="ALL"}(j||(j={})),function(e){e.ASC="ASC",e.DESC="DESC"}(U||(U={})),function(e){e.STANDARD="STANDARD",e.FIFO="FIFO"}(G||(G={}));var Q,z=new Map([[G.STANDARD,"tb.rulenode.sqs-queue-standard"],[G.FIFO,"tb.rulenode.sqs-queue-fifo"]]),$=["anonymous","basic","cert.PEM"],_=new Map([["anonymous","tb.rulenode.credentials-anonymous"],["basic","tb.rulenode.credentials-basic"],["cert.PEM","tb.rulenode.credentials-pem"]]);!function(e){e.GET="GET",e.POST="POST",e.PUT="PUT",e.DELETE="DELETE"}(Q||(Q={}));var W=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(D),n.rangeUnitTranslationMap=B,n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=O,n}return y(r,e),r.prototype.configForm=function(){return this.geoActionConfigForm},r.prototype.onConfigurationSet=function(e){this.geoActionConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]],minInsideDuration:[e?e.minInsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minInsideDurationTimeUnit:[e?e.minInsideDurationTimeUnit:null,[i.Validators.required]],minOutsideDuration:[e?e.minOutsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minOutsideDurationTimeUnit:[e?e.minOutsideDurationTimeUnit:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoActionConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoActionConfigForm.get("perimeterType").value;t?this.geoActionConfigForm.get("perimeterType").setValidators([]):this.geoActionConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoActionConfigForm.get("centerLatitude").setValidators([]),this.geoActionConfigForm.get("centerLongitude").setValidators([]),this.geoActionConfigForm.get("range").setValidators([]),this.geoActionConfigForm.get("rangeUnit").setValidators([])):(this.geoActionConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoActionConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoActionConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoActionConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoActionConfigForm.get("polygonsDefinition").setValidators([]):this.geoActionConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoActionConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoActionConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.min-inside-duration\n \n \n {{ \'tb.rulenode.min-inside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-inside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.min-outside-duration\n \n \n {{ \'tb.rulenode.min-outside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-outside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),J=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgCountConfigForm},r.prototype.onConfigurationSet=function(e){this.msgCountConfigForm=this.fb.group({interval:[e?e.interval:null,[i.Validators.required,i.Validators.min(1)]],telemetryPrefix:[e?e.telemetryPrefix:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-count-config",template:'
\n \n tb.rulenode.interval-seconds\n \n \n {{ \'tb.rulenode.interval-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-interval-seconds-message\' | translate }}\n \n \n \n tb.rulenode.output-timeseries-key-prefix\n \n \n {{ \'tb.rulenode.output-timeseries-key-prefix-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Y=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.rpcReplyConfigForm},r.prototype.onConfigurationSet=function(e){this.rpcReplyConfigForm=this.fb.group({requestIdMetaDataAttribute:[e?e.requestIdMetaDataAttribute:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rpc-reply-config",template:'
\n \n tb.rulenode.request-id-metadata-attribute\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Z=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.saveToCustomTableConfigForm},r.prototype.onConfigurationSet=function(e){this.saveToCustomTableConfigForm=this.fb.group({tableName:[e?e.tableName:null,[i.Validators.required]],fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-custom-table-config",template:'
\n \n tb.rulenode.custom-table-name\n \n \n {{ \'tb.rulenode.custom-table-name-required\' | translate }}\n \n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),X=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.translate=r,o.injector=n,o.fb=a,o.propagateChange=null,o.valueChangeSubscription=null,o}var a;return y(r,e),a=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){this.ngControl=this.injector.get(i.NgControl),null!=this.ngControl&&(this.ngControl.valueAccessor=this),this.kvListFormGroup=this.fb.group({}),this.kvListFormGroup.addControl("keyVals",this.fb.array([]))},r.prototype.keyValsFormArray=function(){return this.kvListFormGroup.get("keyVals")},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.kvListFormGroup.disable({emitEvent:!1}):this.kvListFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t,r,n=this;this.valueChangeSubscription&&this.valueChangeSubscription.unsubscribe();var a=[];if(e)try{for(var o=C(Object.keys(e)),l=o.next();!l.done;l=o.next()){var s=l.value;Object.prototype.hasOwnProperty.call(e,s)&&a.push(this.fb.group({key:[s,[i.Validators.required]],value:[e[s],[i.Validators.required]]}))}}catch(e){t={error:e}}finally{try{l&&!l.done&&(r=o.return)&&r.call(o)}finally{if(t)throw t.error}}this.kvListFormGroup.setControl("keyVals",this.fb.array(a)),this.valueChangeSubscription=this.kvListFormGroup.valueChanges.subscribe((function(){n.updateModel()}))},r.prototype.removeKeyVal=function(e){this.kvListFormGroup.get("keyVals").removeAt(e)},r.prototype.addKeyVal=function(){this.kvListFormGroup.get("keyVals").push(this.fb.group({key:["",[i.Validators.required]],value:["",[i.Validators.required]]}))},r.prototype.validate=function(e){return!this.kvListFormGroup.get("keyVals").value.length&&this.required?{kvMapRequired:!0}:this.kvListFormGroup.valid?null:{kvFieldsRequired:!0}},r.prototype.updateModel=function(){var e=this.kvListFormGroup.get("keyVals").value;if(this.required&&!e.length||!this.kvListFormGroup.valid)this.propagateChange(null);else{var t={};e.forEach((function(e){t[e.key]=e.value})),this.propagateChange(t)}},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:t.Injector},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",String)],r.prototype,"requiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyRequiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valRequiredText",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=a=b([t.Component({selector:"tb-kv-map-config",template:'
\n
\n {{ keyText }}\n {{ valText }}\n \n
\n
\n
\n \n \n \n \n {{ keyRequiredText | translate }}\n \n \n \n \n \n \n {{ valRequiredText | translate }}\n \n \n \n
\n
\n \n
\n \n
\n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return a})),multi:!0},{provide:i.NG_VALIDATORS,useExisting:t.forwardRef((function(){return a})),multi:!0}],styles:[":host .tb-kv-map-config{margin-bottom:16px}:host .tb-kv-map-config .header{padding-left:5px;padding-right:5px;padding-bottom:5px}:host .tb-kv-map-config .header .cell{padding-left:5px;padding-right:5px;color:rgba(0,0,0,.54);font-size:12px;font-weight:700;white-space:nowrap}:host .tb-kv-map-config .body{padding-left:5px;padding-right:5px;padding-bottom:20px;max-height:300px;overflow:auto}:host .tb-kv-map-config .body .row{padding-top:5px;max-height:40px}:host .tb-kv-map-config .body .cell{padding-left:5px;padding-right:5px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell{margin:0;max-height:40px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell .mat-form-field-infix{border-top:0}:host ::ng-deep .tb-kv-map-config .body button.mat-button{margin:0}"]}),h("design:paramtypes",[o.Store,n.TranslateService,t.Injector,i.FormBuilder])],r)}(a.PageComponent),ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.deviceRelationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],relationType:[null],deviceTypes:[null,[i.Validators.required]]}),this.deviceRelationsQueryFormGroup.valueChanges.subscribe((function(t){e.deviceRelationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.deviceRelationsQueryFormGroup.disable({emitEvent:!1}):this.deviceRelationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.deviceRelationsQueryFormGroup.reset(e,{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-device-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-type
\n \n \n
device.device-types
\n \n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),te=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.relationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],filters:[null]}),this.relationsQueryFormGroup.valueChanges.subscribe((function(t){e.relationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.relationsQueryFormGroup.disable({emitEvent:!1}):this.relationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.relationsQueryFormGroup.reset(e||{},{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-filters
\n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),re=function(e){function r(t,r,n,o){var i,l,m=e.call(this,t)||this;m.store=t,m.translate=r,m.truncate=n,m.fb=o,m.placeholder="tb.rulenode.message-type",m.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],m.messageTypes=[],m.messageTypesList=[],m.searchText="",m.propagateChange=function(e){},m.messageTypeConfigForm=m.fb.group({messageType:[null]});try{for(var u=C(Object.keys(a.MessageType)),d=u.next();!d.done;d=u.next()){var p=d.value;m.messageTypesList.push({name:a.messageTypeNames.get(a.MessageType[p]),value:p})}}catch(e){i={error:e}}finally{try{d&&!d.done&&(l=u.return)&&l.call(u)}finally{if(i)throw i.error}}return m}var l;return y(r,e),l=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.ngOnInit=function(){var e=this;this.filteredMessageTypes=this.messageTypeConfigForm.get("messageType").valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(t){return e.fetchMessageTypes(t)})),f.share())},r.prototype.ngAfterViewInit=function(){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.messageTypeConfigForm.disable({emitEvent:!1}):this.messageTypeConfigForm.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t=this;this.searchText="",this.messageTypes.length=0,e&&e.forEach((function(e){var r=t.messageTypesList.find((function(t){return t.value===e}));r?t.messageTypes.push({name:r.name,value:r.value}):t.messageTypes.push({name:e,value:e})}))},r.prototype.displayMessageTypeFn=function(e){return e?e.name:void 0},r.prototype.textIsNotEmpty=function(e){return!!(e&&null!=e&&e.length>0)},r.prototype.createMessageType=function(e,t){e.preventDefault(),this.transformMessageType(t)},r.prototype.add=function(e){this.transformMessageType(e.value)},r.prototype.fetchMessageTypes=function(e){if(this.searchText=e,this.searchText&&this.searchText.length){var t=this.searchText.toUpperCase();return c.of(this.messageTypesList.filter((function(e){return e.name.toUpperCase().includes(t)})))}return c.of(this.messageTypesList)},r.prototype.transformMessageType=function(e){if((e||"").trim()){var t=null,r=e.trim(),n=this.messageTypesList.find((function(e){return e.name===r}));(t=n?{name:n.name,value:n.value}:{name:r,value:r})&&this.addMessageType(t)}this.clear("")},r.prototype.remove=function(e){var t=this.messageTypes.indexOf(e);t>=0&&(this.messageTypes.splice(t,1),this.updateModel())},r.prototype.selected=function(e){this.addMessageType(e.option.value),this.clear("")},r.prototype.addMessageType=function(e){-1===this.messageTypes.findIndex((function(t){return t.value===e.value}))&&(this.messageTypes.push(e),this.updateModel())},r.prototype.onFocus=function(){this.messageTypeConfigForm.get("messageType").updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.messageTypeInput.nativeElement.value=e,this.messageTypeConfigForm.get("messageType").patchValue(null,{emitEvent:!0}),setTimeout((function(){t.messageTypeInput.nativeElement.blur(),t.messageTypeInput.nativeElement.focus()}),0)},r.prototype.updateModel=function(){var e=this.messageTypes.map((function(e){return e.value}));this.required?(this.chipList.errorState=!e.length,this.propagateChange(e.length>0?e:null)):(this.chipList.errorState=!1,this.propagateChange(e))},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:a.TruncatePipe},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),b([t.Input(),h("design:type",String)],r.prototype,"label",void 0),b([t.Input(),h("design:type",Object)],r.prototype,"placeholder",void 0),b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.ViewChild("chipList",{static:!1}),h("design:type",d.MatChipList)],r.prototype,"chipList",void 0),b([t.ViewChild("messageTypeAutocomplete",{static:!1}),h("design:type",p.MatAutocomplete)],r.prototype,"matAutocomplete",void 0),b([t.ViewChild("messageTypeInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"messageTypeInput",void 0),r=l=b([t.Component({selector:"tb-message-types-config",template:'\n {{ label }}\n \n \n {{messageType.name}}\n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-message-types-found\n
\n \n \n {{ translate.get(\'tb.rulenode.no-message-type-matching\',\n {messageType: truncate.transform(searchText, true, 6, '...')}) | async }}\n \n \n \n tb.rulenode.create-new-message-type\n \n
\n \n \n \n {{ \'tb.rulenode.message-types-required\' | translate }}\n \n\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return l})),multi:!0}]}),h("design:paramtypes",[o.Store,n.TranslateService,a.TruncatePipe,i.FormBuilder])],r)}(a.PageComponent),ne=function(){function e(){}return e=b([t.NgModule({declarations:[X,ee,te,re],imports:[r.CommonModule,a.SharedModule,m.HomeComponentsModule],exports:[X,ee,te,re]})],e)}(),ae=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.unassignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.unassignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-un-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),oe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.snsConfigForm},r.prototype.onConfigurationSet=function(e){this.snsConfigForm=this.fb.group({topicArnPattern:[e?e.topicArnPattern:null,[i.Validators.required]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sns-config",template:'
\n \n tb.rulenode.topic-arn-pattern\n \n \n {{ \'tb.rulenode.topic-arn-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.sqsQueueType=G,n.sqsQueueTypes=Object.keys(G),n.sqsQueueTypeTranslationsMap=z,n}return y(r,e),r.prototype.configForm=function(){return this.sqsConfigForm},r.prototype.onConfigurationSet=function(e){this.sqsConfigForm=this.fb.group({queueType:[e?e.queueType:null,[i.Validators.required]],queueUrlPattern:[e?e.queueUrlPattern:null,[i.Validators.required]],delaySeconds:[e?e.delaySeconds:null,[i.Validators.min(0),i.Validators.max(900)]],messageAttributes:[e?e.messageAttributes:null,[]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sqs-config",template:'
\n \n tb.rulenode.queue-type\n \n \n {{ sqsQueueTypeTranslationsMap.get(type) | translate }}\n \n \n \n \n tb.rulenode.queue-url-pattern\n \n \n {{ \'tb.rulenode.queue-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.delay-seconds\n \n \n {{ \'tb.rulenode.min-delay-seconds-message\' | translate }}\n \n \n {{ \'tb.rulenode.max-delay-seconds-message\' | translate }}\n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.pubSubConfigForm},r.prototype.onConfigurationSet=function(e){this.pubSubConfigForm=this.fb.group({projectId:[e?e.projectId:null,[i.Validators.required]],topicName:[e?e.topicName:null,[i.Validators.required]],serviceAccountKey:[e?e.serviceAccountKey:null,[i.Validators.required]],serviceAccountKeyFileName:[e?e.serviceAccountKeyFileName:null,[i.Validators.required]],messageAttributes:[e?e.messageAttributes:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-pub-sub-config",template:'
\n \n tb.rulenode.gcp-project-id\n \n \n {{ \'tb.rulenode.gcp-project-id-required\' | translate }}\n \n \n \n tb.rulenode.pubsub-topic-name\n \n \n {{ \'tb.rulenode.pubsub-topic-name-required\' | translate }}\n \n \n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),se=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.ackValues=["all","-1","0","1"],n}return y(r,e),r.prototype.configForm=function(){return this.kafkaConfigForm},r.prototype.onConfigurationSet=function(e){this.kafkaConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],bootstrapServers:[e?e.bootstrapServers:null,[i.Validators.required]],retries:[e?e.retries:null,[i.Validators.min(0)]],batchSize:[e?e.batchSize:null,[i.Validators.min(0)]],linger:[e?e.linger:null,[i.Validators.min(0)]],bufferMemory:[e?e.bufferMemory:null,[i.Validators.min(0)]],acks:[e?e.acks:null,[i.Validators.required]],keySerializer:[e?e.keySerializer:null,[i.Validators.required]],valueSerializer:[e?e.valueSerializer:null,[i.Validators.required]],otherProperties:[e?e.otherProperties:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-kafka-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n tb.rulenode.bootstrap-servers\n \n \n {{ \'tb.rulenode.bootstrap-servers-required\' | translate }}\n \n \n \n tb.rulenode.retries\n \n \n {{ \'tb.rulenode.min-retries-message\' | translate }}\n \n \n \n tb.rulenode.batch-size-bytes\n \n \n {{ \'tb.rulenode.min-batch-size-bytes-message\' | translate }}\n \n \n \n tb.rulenode.linger-ms\n \n \n {{ \'tb.rulenode.min-linger-ms-message\' | translate }}\n \n \n \n tb.rulenode.buffer-memory-bytes\n \n \n {{ \'tb.rulenode.min-buffer-memory-bytes-message\' | translate }}\n \n \n \n tb.rulenode.acks\n \n \n {{ ackValue }}\n \n \n \n \n tb.rulenode.key-serializer\n \n \n {{ \'tb.rulenode.key-serializer-required\' | translate }}\n \n \n \n tb.rulenode.value-serializer\n \n \n {{ \'tb.rulenode.value-serializer-required\' | translate }}\n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),me=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allMqttCredentialsTypes=$,n.mqttCredentialsTypeTranslationsMap=_,n}return y(r,e),r.prototype.configForm=function(){return this.mqttConfigForm},r.prototype.onConfigurationSet=function(e){this.mqttConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],connectTimeoutSec:[e?e.connectTimeoutSec:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(200)]],clientId:[e?e.clientId:null,[]],cleanSession:[!!e&&e.cleanSession,[]],ssl:[!!e&&e.ssl,[]],credentials:this.fb.group({type:[e&&e.credentials?e.credentials.type:null,[i.Validators.required]],username:[e&&e.credentials?e.credentials.username:null,[]],password:[e&&e.credentials?e.credentials.password:null,[]],caCert:[e&&e.credentials?e.credentials.caCert:null,[]],caCertFileName:[e&&e.credentials?e.credentials.caCertFileName:null,[]],privateKey:[e&&e.credentials?e.credentials.privateKey:null,[]],privateKeyFileName:[e&&e.credentials?e.credentials.privateKeyFileName:null,[]],cert:[e&&e.credentials?e.credentials.cert:null,[]],certFileName:[e&&e.credentials?e.credentials.certFileName:null,[]]})})},r.prototype.prepareOutputConfig=function(e){var t=e.credentials.type;switch(t){case"anonymous":e.credentials={type:t};break;case"basic":e.credentials={type:t,username:e.credentials.username,password:e.credentials.password};break;case"cert.PEM":delete e.credentials.username}return e},r.prototype.validatorTriggers=function(){return["credentials.type"]},r.prototype.updateValidators=function(e){var t=this.mqttConfigForm.get("credentials"),r=t.get("type").value;switch(e&&t.reset({type:r},{emitEvent:!1}),t.get("username").setValidators([]),t.get("password").setValidators([]),t.get("caCert").setValidators([]),t.get("caCertFileName").setValidators([]),t.get("privateKey").setValidators([]),t.get("privateKeyFileName").setValidators([]),t.get("cert").setValidators([]),t.get("certFileName").setValidators([]),r){case"anonymous":break;case"basic":t.get("username").setValidators([i.Validators.required]),t.get("password").setValidators([i.Validators.required]);break;case"cert.PEM":t.get("caCert").setValidators([i.Validators.required]),t.get("caCertFileName").setValidators([i.Validators.required]),t.get("privateKey").setValidators([i.Validators.required]),t.get("privateKeyFileName").setValidators([i.Validators.required]),t.get("cert").setValidators([i.Validators.required]),t.get("certFileName").setValidators([i.Validators.required])}t.get("username").updateValueAndValidity({emitEvent:e}),t.get("password").updateValueAndValidity({emitEvent:e}),t.get("caCert").updateValueAndValidity({emitEvent:e}),t.get("caCertFileName").updateValueAndValidity({emitEvent:e}),t.get("privateKey").updateValueAndValidity({emitEvent:e}),t.get("privateKeyFileName").updateValueAndValidity({emitEvent:e}),t.get("cert").updateValueAndValidity({emitEvent:e}),t.get("certFileName").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-mqtt-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n \n tb.rulenode.connect-timeout\n \n \n {{ \'tb.rulenode.connect-timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n
\n \n tb.rulenode.client-id\n \n \n \n {{ \'tb.rulenode.clean-session\' | translate }}\n \n \n {{ \'tb.rulenode.enable-ssl\' | translate }}\n \n \n \n tb.rulenode.credentials\n \n {{ mqttCredentialsTypeTranslationsMap.get(mqttConfigForm.get(\'credentials\').get(\'type\').value) | translate }}\n \n \n
\n \n tb.rulenode.credentials-type\n \n \n {{ mqttCredentialsTypeTranslationsMap.get(credentialsType) | translate }}\n \n \n \n {{ \'tb.rulenode.credentials-type-required\' | translate }}\n \n \n
\n \n \n \n \n tb.rulenode.username\n \n \n {{ \'tb.rulenode.username-required\' | translate }}\n \n \n \n tb.rulenode.password\n \n \n {{ \'tb.rulenode.password-required\' | translate }}\n \n \n \n \n \n \n \n \n \n \n \n tb.rulenode.private-key-password\n \n \n \n
\n
\n \n
\n',styles:[":host .tb-mqtt-credentials-panel-group{margin:0 6px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ue=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],n}return y(r,e),r.prototype.configForm=function(){return this.rabbitMqConfigForm},r.prototype.onConfigurationSet=function(e){this.rabbitMqConfigForm=this.fb.group({exchangeNamePattern:[e?e.exchangeNamePattern:null,[]],routingKeyPattern:[e?e.routingKeyPattern:null,[]],messageProperties:[e?e.messageProperties:null,[]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],virtualHost:[e?e.virtualHost:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]],automaticRecoveryEnabled:[!!e&&e.automaticRecoveryEnabled,[]],connectionTimeout:[e?e.connectionTimeout:null,[i.Validators.min(0)]],handshakeTimeout:[e?e.handshakeTimeout:null,[i.Validators.min(0)]],clientProperties:[e?e.clientProperties:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rabbit-mq-config",template:'
\n \n tb.rulenode.exchange-name-pattern\n \n \n \n tb.rulenode.routing-key-pattern\n \n \n \n tb.rulenode.message-properties\n \n \n {{ property }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n
\n \n tb.rulenode.virtual-host\n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n \n {{ \'tb.rulenode.automatic-recovery\' | translate }}\n \n \n tb.rulenode.connection-timeout-ms\n \n \n {{ \'tb.rulenode.min-connection-timeout-ms-message\' | translate }}\n \n \n \n tb.rulenode.handshake-timeout-ms\n \n \n {{ \'tb.rulenode.min-handshake-timeout-ms-message\' | translate }}\n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),de=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.httpRequestTypes=Object.keys(Q),n}return y(r,e),r.prototype.configForm=function(){return this.restApiCallConfigForm},r.prototype.onConfigurationSet=function(e){this.restApiCallConfigForm=this.fb.group({restEndpointUrlPattern:[e?e.restEndpointUrlPattern:null,[i.Validators.required]],requestMethod:[e?e.requestMethod:null,[i.Validators.required]],useSimpleClientHttpFactory:[!!e&&e.useSimpleClientHttpFactory,[]],readTimeoutMs:[e?e.readTimeoutMs:null,[]],maxParallelRequestsCount:[e?e.maxParallelRequestsCount:null,[i.Validators.min(0)]],headers:[e?e.headers:null,[]],useRedisQueueForMsgPersistence:[!!e&&e.useRedisQueueForMsgPersistence,[]],trimQueue:[!!e&&e.trimQueue,[]],maxQueueSize:[e?e.maxQueueSize:null,[]]})},r.prototype.validatorTriggers=function(){return["useSimpleClientHttpFactory","useRedisQueueForMsgPersistence"]},r.prototype.updateValidators=function(e){var t=this.restApiCallConfigForm.get("useSimpleClientHttpFactory").value,r=this.restApiCallConfigForm.get("useRedisQueueForMsgPersistence").value;t?this.restApiCallConfigForm.get("readTimeoutMs").setValidators([]):this.restApiCallConfigForm.get("readTimeoutMs").setValidators([i.Validators.min(0)]),r?this.restApiCallConfigForm.get("maxQueueSize").setValidators([i.Validators.min(0)]):this.restApiCallConfigForm.get("maxQueueSize").setValidators([]),this.restApiCallConfigForm.get("readTimeoutMs").updateValueAndValidity({emitEvent:e}),this.restApiCallConfigForm.get("maxQueueSize").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rest-api-call-config",template:'
\n \n tb.rulenode.endpoint-url-pattern\n \n \n {{ \'tb.rulenode.endpoint-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.request-method\n \n \n {{ requestType }}\n \n \n \n \n {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}\n \n \n tb.rulenode.read-timeout\n \n \n \n \n tb.rulenode.max-parallel-requests-count\n \n \n \n \n
tb.rulenode.headers-hint
\n \n \n \n {{ \'tb.rulenode.use-redis-queue\' | translate }}\n \n
\n \n {{ \'tb.rulenode.trim-redis-queue\' | translate }}\n \n \n tb.rulenode.redis-queue-max-size\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),pe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.smtpProtocols=["smtp","smtps"],n.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],n}return y(r,e),r.prototype.configForm=function(){return this.sendEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.sendEmailConfigForm=this.fb.group({useSystemSmtpSettings:[!!e&&e.useSystemSmtpSettings,[]],smtpProtocol:[e?e.smtpProtocol:null,[]],smtpHost:[e?e.smtpHost:null,[]],smtpPort:[e?e.smtpPort:null,[]],timeout:[e?e.timeout:null,[]],enableTls:[!!e&&e.enableTls,[]],tlsVersion:[e?e.tlsVersion:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]]})},r.prototype.validatorTriggers=function(){return["useSystemSmtpSettings"]},r.prototype.updateValidators=function(e){this.sendEmailConfigForm.get("useSystemSmtpSettings").value?(this.sendEmailConfigForm.get("smtpProtocol").setValidators([]),this.sendEmailConfigForm.get("smtpHost").setValidators([]),this.sendEmailConfigForm.get("smtpPort").setValidators([]),this.sendEmailConfigForm.get("timeout").setValidators([])):(this.sendEmailConfigForm.get("smtpProtocol").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpHost").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpPort").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]),this.sendEmailConfigForm.get("timeout").setValidators([i.Validators.required,i.Validators.min(0)])),this.sendEmailConfigForm.get("smtpProtocol").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpHost").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpPort").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("timeout").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-send-email-config",template:'
\n \n {{ \'tb.rulenode.use-system-smtp-settings\' | translate }}\n \n
\n \n tb.rulenode.smtp-protocol\n \n \n {{ smtpProtocol.toUpperCase() }}\n \n \n \n
\n \n tb.rulenode.smtp-host\n \n \n {{ \'tb.rulenode.smtp-host-required\' | translate }}\n \n \n \n tb.rulenode.smtp-port\n \n \n {{ \'tb.rulenode.smtp-port-required\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n
\n \n tb.rulenode.timeout-msec\n \n \n {{ \'tb.rulenode.timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-timeout-msec-message\' | translate }}\n \n \n \n {{ \'tb.rulenode.enable-tls\' | translate }}\n \n \n tb.rulenode.tls-version\n \n \n {{ tlsVersion }}\n \n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ce=function(){function e(){}return e=b([t.NgModule({declarations:[T,x,q,S,I,k,N,V,E,A,L,W,J,Y,Z,ae,oe,ie,le,se,me,ue,de,pe],imports:[r.CommonModule,a.SharedModule,ne],exports:[T,x,q,S,I,k,N,V,E,A,L,W,J,Y,Z,ae,oe,ie,le,se,me,ue,de,pe]})],e)}(),fe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.checkMessageConfigForm},r.prototype.onConfigurationSet=function(e){this.checkMessageConfigForm=this.fb.group({messageNames:[e?e.messageNames:null,[]],metadataNames:[e?e.metadataNames:null,[]],checkAllKeys:[!!e&&e.checkAllKeys,[]]})},r.prototype.validateConfig=function(){var e=this.checkMessageConfigForm.get("messageNames").value,t=this.checkMessageConfigForm.get("metadataNames").value;return e.length>0||t.length>0},r.prototype.removeMessageName=function(e){var t=this.checkMessageConfigForm.get("messageNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("messageNames").setValue(t,{emitEvent:!0}))},r.prototype.removeMetadataName=function(e){var t=this.checkMessageConfigForm.get("metadataNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("metadataNames").setValue(t,{emitEvent:!0}))},r.prototype.addMessageName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("messageNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("messageNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.prototype.addMetadataName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("metadataNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("metadataNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-message-config",template:'
\n \n \n \n \n \n {{messageName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n \n \n \n \n {{metadataName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n {{ \'tb.rulenode.check-all-keys\' | translate }}\n \n
tb.rulenode.check-all-keys-hint
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ge=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.entitySearchDirection=Object.keys(a.EntitySearchDirection),n.entitySearchDirectionTranslationsMap=a.entitySearchDirectionTranslations,n}return y(r,e),r.prototype.configForm=function(){return this.checkRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.checkRelationConfigForm=this.fb.group({checkForSingleEntity:[!!e&&e.checkForSingleEntity,[]],direction:[e?e.direction:null,[]],entityType:[e?e.entityType:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],entityId:[e?e.entityId:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],relationType:[e?e.relationType:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["checkForSingleEntity"]},r.prototype.updateValidators=function(e){var t=this.checkRelationConfigForm.get("checkForSingleEntity").value;this.checkRelationConfigForm.get("entityType").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:e}),this.checkRelationConfigForm.get("entityId").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityId").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-relation-config",template:'
\n \n {{ \'tb.rulenode.check-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.check-relation-hint
\n \n relation.direction\n \n \n {{ entitySearchDirectionTranslationsMap.get(direction) | translate }}\n \n \n \n
\n \n \n \n \n
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ye=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(D),n.rangeUnitTranslationMap=B,n}return y(r,e),r.prototype.configForm=function(){return this.geoFilterConfigForm},r.prototype.onConfigurationSet=function(e){this.geoFilterConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoFilterConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoFilterConfigForm.get("perimeterType").value;t?this.geoFilterConfigForm.get("perimeterType").setValidators([]):this.geoFilterConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoFilterConfigForm.get("centerLatitude").setValidators([]),this.geoFilterConfigForm.get("centerLongitude").setValidators([]),this.geoFilterConfigForm.get("range").setValidators([]),this.geoFilterConfigForm.get("rangeUnit").setValidators([])):(this.geoFilterConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoFilterConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoFilterConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoFilterConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoFilterConfigForm.get("polygonsDefinition").setValidators([]):this.geoFilterConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoFilterConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoFilterConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),be=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.messageTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.messageTypeConfigForm=this.fb.group({messageTypes:[e?e.messageTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-message-type-config",template:'
\n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),he=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allowedEntityTypes=[a.EntityType.DEVICE,a.EntityType.ASSET,a.EntityType.ENTITY_VIEW,a.EntityType.TENANT,a.EntityType.CUSTOMER,a.EntityType.USER,a.EntityType.DASHBOARD,a.EntityType.RULE_CHAIN,a.EntityType.RULE_NODE],n}return y(r,e),r.prototype.configForm=function(){return this.originatorTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorTypeConfigForm=this.fb.group({originatorTypes:[e?e.originatorTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-originator-type-config",template:'
\n \n \n \n
\n',styles:[":host ::ng-deep tb-entity-type-list .mat-form-field-flex{padding-top:0}:host ::ng-deep tb-entity-type-list .mat-form-field-infix{border-top:0}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ce=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"filter",this.translate.instant("tb.rulenode.filter"),"Filter",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),ve=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.switchConfigForm},r.prototype.onConfigurationSet=function(e){this.switchConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.switchConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"switch",this.translate.instant("tb.rulenode.switch"),"Switch",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.switchConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-switch-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),Fe=function(e){function r(t,r,n){var o,l,s=e.call(this,t)||this;s.store=t,s.translate=r,s.fb=n,s.alarmStatusTranslationsMap=a.alarmStatusTranslations,s.alarmStatusList=[],s.searchText="",s.displayStatusFn=s.displayStatus.bind(s);try{for(var m=C(Object.keys(a.AlarmStatus)),u=m.next();!u.done;u=m.next()){var d=u.value;s.alarmStatusList.push(a.AlarmStatus[d])}}catch(e){o={error:e}}finally{try{u&&!u.done&&(l=m.return)&&l.call(m)}finally{if(o)throw o.error}}return s.statusFormControl=new i.FormControl(""),s.filteredAlarmStatus=s.statusFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return s.fetchAlarmStatus(e)})),f.share()),s}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.alarmStatusConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.statusFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.alarmStatusConfigForm=this.fb.group({alarmStatusList:[e?e.alarmStatusList:null,[i.Validators.required]]})},r.prototype.displayStatus=function(e){return e?this.translate.instant(a.alarmStatusTranslations.get(e)):void 0},r.prototype.fetchAlarmStatus=function(e){var t=this,r=this.getAlarmStatusList();if(this.searchText=e,this.searchText&&this.searchText.length){var n=this.searchText.toUpperCase();return c.of(r.filter((function(e){return t.translate.instant(a.alarmStatusTranslations.get(a.AlarmStatus[e])).toUpperCase().includes(n)})))}return c.of(r)},r.prototype.alarmStatusSelected=function(e){this.addAlarmStatus(e.option.value),this.clear("")},r.prototype.removeAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))}},r.prototype.addAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))},r.prototype.getAlarmStatusList=function(){var e=this;return this.alarmStatusList.filter((function(t){return-1===e.alarmStatusConfigForm.get("alarmStatusList").value.indexOf(t)}))},r.prototype.onAlarmStatusInputFocus=function(){this.statusFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.alarmStatusInput.nativeElement.value=e,this.statusFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.alarmStatusInput.nativeElement.blur(),t.alarmStatusInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("alarmStatusInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"alarmStatusInput",void 0),r=b([t.Component({selector:"tb-filter-node-check-alarm-status-config",template:'
\n \n tb.rulenode.alarm-status-filter\n \n \n \n {{alarmStatusTranslationsMap.get(alarmStatus) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-alarm-status-matching\n
\n
\n \n \n \n \n
\n\n\n\n'}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Te=function(){function e(){}return e=b([t.NgModule({declarations:[fe,ge,ye,be,he,Ce,ve,Fe],imports:[r.CommonModule,a.SharedModule,ne],exports:[fe,ge,ye,be,he,Ce,ve,Fe]})],e)}(),xe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.customerAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.customerAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-customer-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),qe=function(e){function r(t,r,n){var a,o,l=e.call(this,t)||this;l.store=t,l.translate=r,l.fb=n,l.entityDetailsTranslationsMap=H,l.entityDetailsList=[],l.searchText="",l.displayDetailsFn=l.displayDetails.bind(l);try{for(var s=C(Object.keys(K)),m=s.next();!m.done;m=s.next()){var u=m.value;l.entityDetailsList.push(K[u])}}catch(e){a={error:e}}finally{try{m&&!m.done&&(o=s.return)&&o.call(s)}finally{if(a)throw a.error}}return l.detailsFormControl=new i.FormControl(""),l.filteredEntityDetails=l.detailsFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return l.fetchEntityDetails(e)})),f.share()),l}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.entityDetailsConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.detailsFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.entityDetailsConfigForm=this.fb.group({detailsList:[e?e.detailsList:null,[i.Validators.required]],addToMetadata:[!!e&&e.addToMetadata,[]]})},r.prototype.displayDetails=function(e){return e?this.translate.instant(H.get(e)):void 0},r.prototype.fetchEntityDetails=function(e){var t=this;if(this.searchText=e,this.searchText&&this.searchText.length){var r=this.searchText.toUpperCase();return c.of(this.entityDetailsList.filter((function(e){return t.translate.instant(H.get(K[e])).toUpperCase().includes(r)})))}return c.of(this.entityDetailsList)},r.prototype.detailsFieldSelected=function(e){this.addDetailsField(e.option.value),this.clear("")},r.prototype.removeDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.entityDetailsConfigForm.get("detailsList").setValue(t))}},r.prototype.addDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.entityDetailsConfigForm.get("detailsList").setValue(t))},r.prototype.onEntityDetailsInputFocus=function(){this.detailsFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.detailsInput.nativeElement.value=e,this.detailsFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.detailsInput.nativeElement.blur(),t.detailsInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("detailsInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"detailsInput",void 0),r=b([t.Component({selector:"tb-enrichment-node-entity-details-config",template:'
\n \n tb.rulenode.entity-details\n \n \n \n {{entityDetailsTranslationsMap.get(details) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-entity-details-matching\n
\n
\n \n \n \n \n \n {{ \'tb.rulenode.add-to-metadata\' | translate }}\n \n
tb.rulenode.add-to-metadata-hint
\n
\n',styles:[":host ::ng-deep mat-form-field.entity-fields-list .mat-form-field-wrapper{margin-bottom:-1.25em}"]}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Se=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.deviceAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.deviceAttributesConfigForm=this.fb.group({deviceRelationsQuery:[e?e.deviceRelationsQuery:null,[i.Validators.required]],tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.deviceAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.deviceAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.deviceAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.deviceAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-device-attributes-config",template:'
\n \n \n \n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.originatorAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorAttributesConfigForm=this.fb.group({tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.originatorAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.originatorAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.originatorAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.originatorAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-attributes-config",template:'
\n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ke=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.originatorFieldsConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorFieldsConfigForm=this.fb.group({fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-fields-config",template:'
\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ne=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n.fetchMode=j,n.fetchModes=Object.keys(j),n.samplingOrders=Object.keys(U),n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=O,n}return y(r,e),r.prototype.configForm=function(){return this.getTelemetryFromDatabaseConfigForm},r.prototype.onConfigurationSet=function(e){this.getTelemetryFromDatabaseConfigForm=this.fb.group({latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],fetchMode:[e?e.fetchMode:null,[i.Validators.required]],orderBy:[e?e.orderBy:null,[]],limit:[e?e.limit:null,[]],useMetadataIntervalPatterns:[!!e&&e.useMetadataIntervalPatterns,[]],startInterval:[e?e.startInterval:null,[]],startIntervalTimeUnit:[e?e.startIntervalTimeUnit:null,[]],endInterval:[e?e.endInterval:null,[]],endIntervalTimeUnit:[e?e.endIntervalTimeUnit:null,[]],startIntervalPattern:[e?e.startIntervalPattern:null,[]],endIntervalPattern:[e?e.endIntervalPattern:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchMode","useMetadataIntervalPatterns"]},r.prototype.updateValidators=function(e){var t=this.getTelemetryFromDatabaseConfigForm.get("fetchMode").value,r=this.getTelemetryFromDatabaseConfigForm.get("useMetadataIntervalPatterns").value;t&&t===j.ALL?(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([i.Validators.required,i.Validators.min(2),i.Validators.max(1e3)])):(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([])),r?(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([i.Validators.required])):(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([])),this.getTelemetryFromDatabaseConfigForm.get("orderBy").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("limit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").updateValueAndValidity({emitEvent:e})},r.prototype.removeKey=function(e,t){var r=this.getTelemetryFromDatabaseConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.getTelemetryFromDatabaseConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-get-telemetry-from-database",template:'
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n tb.rulenode.fetch-mode\n \n \n {{ mode }}\n \n \n tb.rulenode.fetch-mode-hint\n \n
\n \n tb.rulenode.order-by\n \n \n {{ order }}\n \n \n tb.rulenode.order-by-hint\n \n \n tb.rulenode.limit\n \n tb.rulenode.limit-hint\n \n
\n \n {{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-interval-patterns-hint
\n
\n
\n \n tb.rulenode.start-interval\n \n \n {{ \'tb.rulenode.start-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.start-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.end-interval\n \n \n {{ \'tb.rulenode.end-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.end-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n \n tb.rulenode.start-interval-pattern\n \n \n {{ \'tb.rulenode.start-interval-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.end-interval-pattern\n \n \n {{ \'tb.rulenode.end-interval-pattern-required\' | translate }}\n \n \n \n \n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ve=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.relatedAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.relatedAttributesConfigForm=this.fb.group({relationsQuery:[e?e.relationsQuery:null,[i.Validators.required]],telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-related-attributes-config",template:'
\n \n \n \n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.tenantAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.tenantAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-tenant-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ae=function(){function e(){}return e=b([t.NgModule({declarations:[xe,qe,Se,Ie,ke,Ne,Ve,Ee],imports:[r.CommonModule,a.SharedModule,ne],exports:[xe,qe,Se,Ie,ke,Ne,Ve,Ee]})],e)}(),Le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.originatorSource=v,n.originatorSources=Object.keys(v),n.originatorSourceTranslationMap=P,n}return y(r,e),r.prototype.configForm=function(){return this.changeOriginatorConfigForm},r.prototype.onConfigurationSet=function(e){this.changeOriginatorConfigForm=this.fb.group({originatorSource:[e?e.originatorSource:null,[i.Validators.required]],relationsQuery:[e?e.relationsQuery:null,[]]})},r.prototype.validatorTriggers=function(){return["originatorSource"]},r.prototype.updateValidators=function(e){var t=this.changeOriginatorConfigForm.get("originatorSource").value;t&&t===v.RELATED?this.changeOriginatorConfigForm.get("relationsQuery").setValidators([i.Validators.required]):this.changeOriginatorConfigForm.get("relationsQuery").setValidators([]),this.changeOriginatorConfigForm.get("relationsQuery").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-change-originator-config",template:'
\n \n tb.rulenode.originator-source\n \n \n {{ originatorSourceTranslationMap.get(source) | translate }}\n \n \n \n
\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Me=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"update",this.translate.instant("tb.rulenode.transformer"),"Transform",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-transformation-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),Pe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.toEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.toEmailConfigForm=this.fb.group({fromTemplate:[e?e.fromTemplate:null,[i.Validators.required]],toTemplate:[e?e.toTemplate:null,[i.Validators.required]],ccTemplate:[e?e.ccTemplate:null,[]],bccTemplate:[e?e.bccTemplate:null,[]],subjectTemplate:[e?e.subjectTemplate:null,[i.Validators.required]],bodyTemplate:[e?e.bodyTemplate:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-to-email-config",template:'
\n \n tb.rulenode.from-template\n \n \n {{ \'tb.rulenode.from-template-required\' | translate }}\n \n \n \n \n tb.rulenode.to-template\n \n \n {{ \'tb.rulenode.to-template-required\' | translate }}\n \n \n \n \n tb.rulenode.cc-template\n \n \n \n \n tb.rulenode.bcc-template\n \n \n \n \n tb.rulenode.subject-template\n \n \n {{ \'tb.rulenode.subject-template-required\' | translate }}\n \n \n \n \n tb.rulenode.body-template\n \n \n {{ \'tb.rulenode.body-template-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Re=function(){function e(){}return e=b([t.NgModule({declarations:[Le,Me,Pe],imports:[r.CommonModule,a.SharedModule,ne],exports:[Le,Me,Pe]})],e)}(),we=function(){function e(e){!function(e){e.setTranslation("en_US",{tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-status-filter":"Alarm status filter","alarm-status-list-empty":"Alarm status list is empty","no-alarm-status-matching":"No alarm status matching were found.",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size"},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}},!0)}(e)}return e.ctorParameters=function(){return[{type:n.TranslateService}]},e=b([t.NgModule({declarations:[F],imports:[r.CommonModule,a.SharedModule],exports:[ce,Te,Ae,Re,F]}),h("design:paramtypes",[n.TranslateService])],e)}();e.RuleNodeCoreConfigModule=we,e.ɵa=F,e.ɵb=ce,e.ɵba=ne,e.ɵbb=X,e.ɵbc=ee,e.ɵbd=te,e.ɵbe=re,e.ɵbf=Te,e.ɵbg=fe,e.ɵbh=ge,e.ɵbi=ye,e.ɵbj=be,e.ɵbk=he,e.ɵbl=Ce,e.ɵbm=ve,e.ɵbn=Fe,e.ɵbo=Ae,e.ɵbp=xe,e.ɵbq=qe,e.ɵbr=Se,e.ɵbs=Ie,e.ɵbt=ke,e.ɵbu=Ne,e.ɵbv=Ve,e.ɵbw=Ee,e.ɵbx=Re,e.ɵby=Le,e.ɵbz=Me,e.ɵc=T,e.ɵca=Pe,e.ɵd=x,e.ɵe=q,e.ɵf=S,e.ɵg=I,e.ɵh=k,e.ɵi=N,e.ɵj=V,e.ɵk=E,e.ɵl=A,e.ɵm=L,e.ɵn=W,e.ɵo=J,e.ɵp=Y,e.ɵq=Z,e.ɵr=ae,e.ɵs=oe,e.ɵt=ie,e.ɵu=le,e.ɵv=se,e.ɵw=me,e.ɵx=ue,e.ɵy=de,e.ɵz=pe,Object.defineProperty(e,"__esModule",{value:!0})})); + ***************************************************************************** */var g=function(e,t){return(g=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var r in t)t.hasOwnProperty(r)&&(e[r]=t[r])})(e,t)};function y(e,t){function r(){this.constructor=e}g(e,t),e.prototype=null===t?Object.create(t):(r.prototype=t.prototype,new r)}function b(e,t,r,n){var a,o=arguments.length,i=o<3?t:null===n?n=Object.getOwnPropertyDescriptor(t,r):n;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)i=Reflect.decorate(e,t,r,n);else for(var l=e.length-1;l>=0;l--)(a=e[l])&&(i=(o<3?a(i):o>3?a(t,r,i):a(t,r))||i);return o>3&&i&&Object.defineProperty(t,r,i),i}function h(e,t){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(e,t)}function C(e){var t="function"==typeof Symbol&&Symbol.iterator,r=t&&e[t],n=0;if(r)return r.call(e);if(e&&"number"==typeof e.length)return{next:function(){return e&&n>=e.length&&(e=void 0),{value:e&&e[n++],done:!e}}};throw new TypeError(t?"Object is not iterable.":"Symbol.iterator is not defined.")}var v,F=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.emptyConfigForm},r.prototype.onConfigurationSet=function(e){this.emptyConfigForm=this.fb.group({})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-node-empty-config",template:"
"}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),T=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.attributeScopes=Object.keys(a.AttributeScope),n.telemetryTypeTranslationsMap=a.telemetryTypeTranslations,n}return y(r,e),r.prototype.configForm=function(){return this.attributesConfigForm},r.prototype.onConfigurationSet=function(e){this.attributesConfigForm=this.fb.group({scope:[e?e.scope:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-attributes-config",template:'
\n \n attribute.attributes-scope\n \n \n {{ telemetryTypeTranslationsMap.get(scope) | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),x=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.timeseriesConfigForm},r.prototype.onConfigurationSet=function(e){this.timeseriesConfigForm=this.fb.group({defaultTTL:[e?e.defaultTTL:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-timeseries-config",template:'
\n \n tb.rulenode.default-ttl\n \n \n {{ \'tb.rulenode.default-ttl-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-default-ttl-message\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),q=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.rpcRequestConfigForm},r.prototype.onConfigurationSet=function(e){this.rpcRequestConfigForm=this.fb.group({timeoutInSeconds:[e?e.timeoutInSeconds:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rpc-request-config",template:'
\n \n tb.rulenode.timeout-sec\n \n \n {{ \'tb.rulenode.timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-timeout-message\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),S=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.logConfigForm},r.prototype.onConfigurationSet=function(e){this.logConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.logConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"string",this.translate.instant("tb.rulenode.to-string"),"ToString",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.logConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-log-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),I=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.assignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.assignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],createCustomerIfNotExists:[!!e&&e.createCustomerIfNotExists,[]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n {{ \'tb.rulenode.create-customer-if-not-exists\' | translate }}\n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),k=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.clearAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.clearAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],alarmType:[e?e.alarmType:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.clearAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.clearAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-clear-alarm-config",template:'
\n \n \n \n
\n \n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),N=function(e){function r(t,r,n,o){var i=e.call(this,t)||this;return i.store=t,i.fb=r,i.nodeScriptTestService=n,i.translate=o,i.alarmSeverities=Object.keys(a.AlarmSeverity),i.alarmSeverityTranslationMap=a.alarmSeverityTranslations,i.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],i}return y(r,e),r.prototype.configForm=function(){return this.createAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.createAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],useMessageAlarmData:[!!e&&e.useMessageAlarmData,[]],alarmType:[e?e.alarmType:null,[]],severity:[e?e.severity:null,[]],propagate:[!!e&&e.propagate,[]],relationTypes:[e?e.relationTypes:null,[]]})},r.prototype.validatorTriggers=function(){return["useMessageAlarmData"]},r.prototype.updateValidators=function(e){this.createAlarmConfigForm.get("useMessageAlarmData").value?(this.createAlarmConfigForm.get("alarmType").setValidators([]),this.createAlarmConfigForm.get("severity").setValidators([])):(this.createAlarmConfigForm.get("alarmType").setValidators([i.Validators.required]),this.createAlarmConfigForm.get("severity").setValidators([i.Validators.required])),this.createAlarmConfigForm.get("alarmType").updateValueAndValidity({emitEvent:e}),this.createAlarmConfigForm.get("severity").updateValueAndValidity({emitEvent:e})},r.prototype.testScript=function(){var e=this,t=this.createAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.createAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.removeKey=function(e,t){var r=this.createAlarmConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.createAlarmConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.createAlarmConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.createAlarmConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-create-alarm-config",template:'
\n \n \n \n
\n \n
\n \n {{ \'tb.rulenode.use-message-alarm-data\' | translate }}\n \n
\n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n \n tb.rulenode.alarm-severity\n \n \n {{ alarmSeverityTranslationMap.get(severity) | translate }}\n \n \n \n {{ \'tb.rulenode.alarm-severity-required\' | translate }}\n \n \n
\n \n {{ \'tb.rulenode.propagate\' | translate }}\n \n
\n \n tb.rulenode.relation-types-list\n \n \n {{key}}\n close\n \n \n \n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),V=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.createRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.createRelationConfigForm=this.fb.group({direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[i.Validators.required]],entityNamePattern:[e?e.entityNamePattern:null,[]],entityTypePattern:[e?e.entityTypePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],createEntityIfNotExists:[!!e&&e.createEntityIfNotExists,[]],removeCurrentRelations:[!!e&&e.removeCurrentRelations,[]],changeOriginatorToRelatedEntity:[!!e&&e.changeOriginatorToRelatedEntity,[]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["entityType"]},r.prototype.updateValidators=function(e){var t=this.createRelationConfigForm.get("entityType").value;t?this.createRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.createRelationConfigForm.get("entityNamePattern").setValidators([]),!t||t!==a.EntityType.DEVICE&&t!==a.EntityType.ASSET?this.createRelationConfigForm.get("entityTypePattern").setValidators([]):this.createRelationConfigForm.get("entityTypePattern").setValidators([i.Validators.required]),this.createRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e}),this.createRelationConfigForm.get("entityTypePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-create-relation-config",template:'
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-type-pattern\n \n \n {{ \'tb.rulenode.entity-type-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n
\n \n {{ \'tb.rulenode.create-entity-if-not-exists\' | translate }}\n \n
tb.rulenode.create-entity-if-not-exists-hint
\n
\n \n {{ \'tb.rulenode.remove-current-relations\' | translate }}\n \n
tb.rulenode.remove-current-relations-hint
\n \n {{ \'tb.rulenode.change-originator-to-related-entity\' | translate }}\n \n
tb.rulenode.change-originator-to-related-entity-hint
\n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),E=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgDelayConfigForm},r.prototype.onConfigurationSet=function(e){this.msgDelayConfigForm=this.fb.group({useMetadataPeriodInSecondsPatterns:[!!e&&e.useMetadataPeriodInSecondsPatterns,[]],periodInSeconds:[e?e.periodInSeconds:null,[]],periodInSecondsPattern:[e?e.periodInSecondsPattern:null,[]],maxPendingMsgs:[e?e.maxPendingMsgs:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(1e5)]]})},r.prototype.validatorTriggers=function(){return["useMetadataPeriodInSecondsPatterns"]},r.prototype.updateValidators=function(e){this.msgDelayConfigForm.get("useMetadataPeriodInSecondsPatterns").value?(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([i.Validators.required]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([])):(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([i.Validators.required,i.Validators.min(0)])),this.msgDelayConfigForm.get("periodInSecondsPattern").updateValueAndValidity({emitEvent:e}),this.msgDelayConfigForm.get("periodInSeconds").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-delay-config",template:'
\n \n {{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
\n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-0-seconds-message\' | translate }}\n \n \n \n \n tb.rulenode.period-in-seconds-pattern\n \n \n {{ \'tb.rulenode.period-in-seconds-pattern-required\' | translate }}\n \n \n \n \n \n tb.rulenode.max-pending-messages\n \n \n {{ \'tb.rulenode.max-pending-messages-required\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),A=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.deleteRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.deleteRelationConfigForm=this.fb.group({deleteForSingleEntity:[!!e&&e.deleteForSingleEntity,[]],direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[]],entityNamePattern:[e?e.entityNamePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["deleteForSingleEntity","entityType"]},r.prototype.updateValidators=function(e){var t=this.deleteRelationConfigForm.get("deleteForSingleEntity").value,r=this.deleteRelationConfigForm.get("entityType").value;t?this.deleteRelationConfigForm.get("entityType").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityType").setValidators([]),t&&r?this.deleteRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityNamePattern").setValidators([]),this.deleteRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:!1}),this.deleteRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-delete-relation-config",template:'
\n \n {{ \'tb.rulenode.delete-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.delete-relation-hint
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),L=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.generatorConfigForm},r.prototype.onConfigurationSet=function(e){this.generatorConfigForm=this.fb.group({msgCount:[e?e.msgCount:null,[i.Validators.required,i.Validators.min(0)]],periodInSeconds:[e?e.periodInSeconds:null,[i.Validators.required,i.Validators.min(1)]],originator:[e?e.originator:null,[]],jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.prepareInputConfig=function(e){return e&&(e.originatorId&&e.originatorType?e.originator={id:e.originatorId,entityType:e.originatorType}:e.originator=null,delete e.originatorId,delete e.originatorType),e},r.prototype.prepareOutputConfig=function(e){return e.originator?(e.originatorId=e.originator.id,e.originatorType=e.originator.entityType):(e.originatorId=null,e.originatorType=null),delete e.originator,e},r.prototype.testScript=function(){var e=this,t=this.generatorConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"generate",this.translate.instant("tb.rulenode.generator"),"Generate",["prevMsg","prevMetadata","prevMsgType"],this.ruleNodeId).subscribe((function(t){t&&e.generatorConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-generator-config",template:'
\n \n tb.rulenode.message-count\n \n \n {{ \'tb.rulenode.message-count-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-message-count-message\' | translate }}\n \n \n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-seconds-message\' | translate }}\n \n \n
\n \n \n \n
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent);!function(e){e.CUSTOMER="CUSTOMER",e.TENANT="TENANT",e.RELATED="RELATED",e.ALARM_ORIGINATOR="ALARM_ORIGINATOR"}(v||(v={}));var M,P=new Map([[v.CUSTOMER,"tb.rulenode.originator-customer"],[v.TENANT,"tb.rulenode.originator-tenant"],[v.RELATED,"tb.rulenode.originator-related"],[v.ALARM_ORIGINATOR,"tb.rulenode.originator-alarm-originator"]]);!function(e){e.CIRCLE="CIRCLE",e.POLYGON="POLYGON"}(M||(M={}));var R,w=new Map([[M.CIRCLE,"tb.rulenode.perimeter-circle"],[M.POLYGON,"tb.rulenode.perimeter-polygon"]]);!function(e){e.MILLISECONDS="MILLISECONDS",e.SECONDS="SECONDS",e.MINUTES="MINUTES",e.HOURS="HOURS",e.DAYS="DAYS"}(R||(R={}));var O,D=new Map([[R.MILLISECONDS,"tb.rulenode.time-unit-milliseconds"],[R.SECONDS,"tb.rulenode.time-unit-seconds"],[R.MINUTES,"tb.rulenode.time-unit-minutes"],[R.HOURS,"tb.rulenode.time-unit-hours"],[R.DAYS,"tb.rulenode.time-unit-days"]]);!function(e){e.METER="METER",e.KILOMETER="KILOMETER",e.FOOT="FOOT",e.MILE="MILE",e.NAUTICAL_MILE="NAUTICAL_MILE"}(O||(O={}));var K,B=new Map([[O.METER,"tb.rulenode.range-unit-meter"],[O.KILOMETER,"tb.rulenode.range-unit-kilometer"],[O.FOOT,"tb.rulenode.range-unit-foot"],[O.MILE,"tb.rulenode.range-unit-mile"],[O.NAUTICAL_MILE,"tb.rulenode.range-unit-nautical-mile"]]);!function(e){e.TITLE="TITLE",e.COUNTRY="COUNTRY",e.STATE="STATE",e.ZIP="ZIP",e.ADDRESS="ADDRESS",e.ADDRESS2="ADDRESS2",e.PHONE="PHONE",e.EMAIL="EMAIL",e.ADDITIONAL_INFO="ADDITIONAL_INFO"}(K||(K={}));var j,U,G,H=new Map([[K.TITLE,"tb.rulenode.entity-details-title"],[K.COUNTRY,"tb.rulenode.entity-details-country"],[K.STATE,"tb.rulenode.entity-details-state"],[K.ZIP,"tb.rulenode.entity-details-zip"],[K.ADDRESS,"tb.rulenode.entity-details-address"],[K.ADDRESS2,"tb.rulenode.entity-details-address2"],[K.PHONE,"tb.rulenode.entity-details-phone"],[K.EMAIL,"tb.rulenode.entity-details-email"],[K.ADDITIONAL_INFO,"tb.rulenode.entity-details-additional_info"]]);!function(e){e.FIRST="FIRST",e.LAST="LAST",e.ALL="ALL"}(j||(j={})),function(e){e.ASC="ASC",e.DESC="DESC"}(U||(U={})),function(e){e.STANDARD="STANDARD",e.FIFO="FIFO"}(G||(G={}));var Q,z=new Map([[G.STANDARD,"tb.rulenode.sqs-queue-standard"],[G.FIFO,"tb.rulenode.sqs-queue-fifo"]]),$=["anonymous","basic","cert.PEM"],_=new Map([["anonymous","tb.rulenode.credentials-anonymous"],["basic","tb.rulenode.credentials-basic"],["cert.PEM","tb.rulenode.credentials-pem"]]);!function(e){e.GET="GET",e.POST="POST",e.PUT="PUT",e.DELETE="DELETE"}(Q||(Q={}));var W=["US-ASCII","ISO-8859-1","UTF-8","UTF-16BE","UTF-16LE","UTF-16"],J=new Map([["US-ASCII","tb.rulenode.charset-us-ascii"],["ISO-8859-1","tb.rulenode.charset-iso-8859-1"],["UTF-8","tb.rulenode.charset-utf-8"],["UTF-16BE","tb.rulenode.charset-utf-16be"],["UTF-16LE","tb.rulenode.charset-utf-16le"],["UTF-16","tb.rulenode.charset-utf-16"]]),Y=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(O),n.rangeUnitTranslationMap=B,n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=D,n}return y(r,e),r.prototype.configForm=function(){return this.geoActionConfigForm},r.prototype.onConfigurationSet=function(e){this.geoActionConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]],minInsideDuration:[e?e.minInsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minInsideDurationTimeUnit:[e?e.minInsideDurationTimeUnit:null,[i.Validators.required]],minOutsideDuration:[e?e.minOutsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minOutsideDurationTimeUnit:[e?e.minOutsideDurationTimeUnit:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoActionConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoActionConfigForm.get("perimeterType").value;t?this.geoActionConfigForm.get("perimeterType").setValidators([]):this.geoActionConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoActionConfigForm.get("centerLatitude").setValidators([]),this.geoActionConfigForm.get("centerLongitude").setValidators([]),this.geoActionConfigForm.get("range").setValidators([]),this.geoActionConfigForm.get("rangeUnit").setValidators([])):(this.geoActionConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoActionConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoActionConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoActionConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoActionConfigForm.get("polygonsDefinition").setValidators([]):this.geoActionConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoActionConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoActionConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.min-inside-duration\n \n \n {{ \'tb.rulenode.min-inside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-inside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.min-outside-duration\n \n \n {{ \'tb.rulenode.min-outside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-outside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Z=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgCountConfigForm},r.prototype.onConfigurationSet=function(e){this.msgCountConfigForm=this.fb.group({interval:[e?e.interval:null,[i.Validators.required,i.Validators.min(1)]],telemetryPrefix:[e?e.telemetryPrefix:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-count-config",template:'
\n \n tb.rulenode.interval-seconds\n \n \n {{ \'tb.rulenode.interval-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-interval-seconds-message\' | translate }}\n \n \n \n tb.rulenode.output-timeseries-key-prefix\n \n \n {{ \'tb.rulenode.output-timeseries-key-prefix-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),X=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.rpcReplyConfigForm},r.prototype.onConfigurationSet=function(e){this.rpcReplyConfigForm=this.fb.group({requestIdMetaDataAttribute:[e?e.requestIdMetaDataAttribute:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rpc-reply-config",template:'
\n \n tb.rulenode.request-id-metadata-attribute\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.saveToCustomTableConfigForm},r.prototype.onConfigurationSet=function(e){this.saveToCustomTableConfigForm=this.fb.group({tableName:[e?e.tableName:null,[i.Validators.required]],fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-custom-table-config",template:'
\n \n tb.rulenode.custom-table-name\n \n \n {{ \'tb.rulenode.custom-table-name-required\' | translate }}\n \n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),te=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.translate=r,o.injector=n,o.fb=a,o.propagateChange=null,o.valueChangeSubscription=null,o}var a;return y(r,e),a=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){this.ngControl=this.injector.get(i.NgControl),null!=this.ngControl&&(this.ngControl.valueAccessor=this),this.kvListFormGroup=this.fb.group({}),this.kvListFormGroup.addControl("keyVals",this.fb.array([]))},r.prototype.keyValsFormArray=function(){return this.kvListFormGroup.get("keyVals")},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.kvListFormGroup.disable({emitEvent:!1}):this.kvListFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t,r,n=this;this.valueChangeSubscription&&this.valueChangeSubscription.unsubscribe();var a=[];if(e)try{for(var o=C(Object.keys(e)),l=o.next();!l.done;l=o.next()){var s=l.value;Object.prototype.hasOwnProperty.call(e,s)&&a.push(this.fb.group({key:[s,[i.Validators.required]],value:[e[s],[i.Validators.required]]}))}}catch(e){t={error:e}}finally{try{l&&!l.done&&(r=o.return)&&r.call(o)}finally{if(t)throw t.error}}this.kvListFormGroup.setControl("keyVals",this.fb.array(a)),this.valueChangeSubscription=this.kvListFormGroup.valueChanges.subscribe((function(){n.updateModel()}))},r.prototype.removeKeyVal=function(e){this.kvListFormGroup.get("keyVals").removeAt(e)},r.prototype.addKeyVal=function(){this.kvListFormGroup.get("keyVals").push(this.fb.group({key:["",[i.Validators.required]],value:["",[i.Validators.required]]}))},r.prototype.validate=function(e){return!this.kvListFormGroup.get("keyVals").value.length&&this.required?{kvMapRequired:!0}:this.kvListFormGroup.valid?null:{kvFieldsRequired:!0}},r.prototype.updateModel=function(){var e=this.kvListFormGroup.get("keyVals").value;if(this.required&&!e.length||!this.kvListFormGroup.valid)this.propagateChange(null);else{var t={};e.forEach((function(e){t[e.key]=e.value})),this.propagateChange(t)}},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:t.Injector},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",String)],r.prototype,"requiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyRequiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valRequiredText",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=a=b([t.Component({selector:"tb-kv-map-config",template:'
\n
\n {{ keyText }}\n {{ valText }}\n \n
\n
\n
\n \n \n \n \n {{ keyRequiredText | translate }}\n \n \n \n \n \n \n {{ valRequiredText | translate }}\n \n \n \n
\n
\n \n
\n \n
\n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return a})),multi:!0},{provide:i.NG_VALIDATORS,useExisting:t.forwardRef((function(){return a})),multi:!0}],styles:[":host .tb-kv-map-config{margin-bottom:16px}:host .tb-kv-map-config .header{padding-left:5px;padding-right:5px;padding-bottom:5px}:host .tb-kv-map-config .header .cell{padding-left:5px;padding-right:5px;color:rgba(0,0,0,.54);font-size:12px;font-weight:700;white-space:nowrap}:host .tb-kv-map-config .body{padding-left:5px;padding-right:5px;padding-bottom:20px;max-height:300px;overflow:auto}:host .tb-kv-map-config .body .row{padding-top:5px;max-height:40px}:host .tb-kv-map-config .body .cell{padding-left:5px;padding-right:5px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell{margin:0;max-height:40px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell .mat-form-field-infix{border-top:0}:host ::ng-deep .tb-kv-map-config .body button.mat-button{margin:0}"]}),h("design:paramtypes",[o.Store,n.TranslateService,t.Injector,i.FormBuilder])],r)}(a.PageComponent),re=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.deviceRelationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],relationType:[null],deviceTypes:[null,[i.Validators.required]]}),this.deviceRelationsQueryFormGroup.valueChanges.subscribe((function(t){e.deviceRelationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.deviceRelationsQueryFormGroup.disable({emitEvent:!1}):this.deviceRelationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.deviceRelationsQueryFormGroup.reset(e,{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-device-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-type
\n \n \n
device.device-types
\n \n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),ne=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.relationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],filters:[null]}),this.relationsQueryFormGroup.valueChanges.subscribe((function(t){e.relationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.relationsQueryFormGroup.disable({emitEvent:!1}):this.relationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.relationsQueryFormGroup.reset(e||{},{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-filters
\n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),ae=function(e){function r(t,r,n,o){var i,l,m=e.call(this,t)||this;m.store=t,m.translate=r,m.truncate=n,m.fb=o,m.placeholder="tb.rulenode.message-type",m.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],m.messageTypes=[],m.messageTypesList=[],m.searchText="",m.propagateChange=function(e){},m.messageTypeConfigForm=m.fb.group({messageType:[null]});try{for(var u=C(Object.keys(a.MessageType)),d=u.next();!d.done;d=u.next()){var p=d.value;m.messageTypesList.push({name:a.messageTypeNames.get(a.MessageType[p]),value:p})}}catch(e){i={error:e}}finally{try{d&&!d.done&&(l=u.return)&&l.call(u)}finally{if(i)throw i.error}}return m}var l;return y(r,e),l=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.ngOnInit=function(){var e=this;this.filteredMessageTypes=this.messageTypeConfigForm.get("messageType").valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(t){return e.fetchMessageTypes(t)})),f.share())},r.prototype.ngAfterViewInit=function(){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.messageTypeConfigForm.disable({emitEvent:!1}):this.messageTypeConfigForm.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t=this;this.searchText="",this.messageTypes.length=0,e&&e.forEach((function(e){var r=t.messageTypesList.find((function(t){return t.value===e}));r?t.messageTypes.push({name:r.name,value:r.value}):t.messageTypes.push({name:e,value:e})}))},r.prototype.displayMessageTypeFn=function(e){return e?e.name:void 0},r.prototype.textIsNotEmpty=function(e){return!!(e&&null!=e&&e.length>0)},r.prototype.createMessageType=function(e,t){e.preventDefault(),this.transformMessageType(t)},r.prototype.add=function(e){this.transformMessageType(e.value)},r.prototype.fetchMessageTypes=function(e){if(this.searchText=e,this.searchText&&this.searchText.length){var t=this.searchText.toUpperCase();return c.of(this.messageTypesList.filter((function(e){return e.name.toUpperCase().includes(t)})))}return c.of(this.messageTypesList)},r.prototype.transformMessageType=function(e){if((e||"").trim()){var t=null,r=e.trim(),n=this.messageTypesList.find((function(e){return e.name===r}));(t=n?{name:n.name,value:n.value}:{name:r,value:r})&&this.addMessageType(t)}this.clear("")},r.prototype.remove=function(e){var t=this.messageTypes.indexOf(e);t>=0&&(this.messageTypes.splice(t,1),this.updateModel())},r.prototype.selected=function(e){this.addMessageType(e.option.value),this.clear("")},r.prototype.addMessageType=function(e){-1===this.messageTypes.findIndex((function(t){return t.value===e.value}))&&(this.messageTypes.push(e),this.updateModel())},r.prototype.onFocus=function(){this.messageTypeConfigForm.get("messageType").updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.messageTypeInput.nativeElement.value=e,this.messageTypeConfigForm.get("messageType").patchValue(null,{emitEvent:!0}),setTimeout((function(){t.messageTypeInput.nativeElement.blur(),t.messageTypeInput.nativeElement.focus()}),0)},r.prototype.updateModel=function(){var e=this.messageTypes.map((function(e){return e.value}));this.required?(this.chipList.errorState=!e.length,this.propagateChange(e.length>0?e:null)):(this.chipList.errorState=!1,this.propagateChange(e))},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:a.TruncatePipe},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),b([t.Input(),h("design:type",String)],r.prototype,"label",void 0),b([t.Input(),h("design:type",Object)],r.prototype,"placeholder",void 0),b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.ViewChild("chipList",{static:!1}),h("design:type",d.MatChipList)],r.prototype,"chipList",void 0),b([t.ViewChild("messageTypeAutocomplete",{static:!1}),h("design:type",p.MatAutocomplete)],r.prototype,"matAutocomplete",void 0),b([t.ViewChild("messageTypeInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"messageTypeInput",void 0),r=l=b([t.Component({selector:"tb-message-types-config",template:'\n {{ label }}\n \n \n {{messageType.name}}\n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-message-types-found\n
\n \n \n {{ translate.get(\'tb.rulenode.no-message-type-matching\',\n {messageType: truncate.transform(searchText, true, 6, '...')}) | async }}\n \n \n \n tb.rulenode.create-new-message-type\n \n
\n \n \n \n {{ \'tb.rulenode.message-types-required\' | translate }}\n \n\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return l})),multi:!0}]}),h("design:paramtypes",[o.Store,n.TranslateService,a.TruncatePipe,i.FormBuilder])],r)}(a.PageComponent),oe=function(){function e(){}return e=b([t.NgModule({declarations:[te,re,ne,ae],imports:[r.CommonModule,a.SharedModule,m.HomeComponentsModule],exports:[te,re,ne,ae]})],e)}(),ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.unassignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.unassignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-un-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.snsConfigForm},r.prototype.onConfigurationSet=function(e){this.snsConfigForm=this.fb.group({topicArnPattern:[e?e.topicArnPattern:null,[i.Validators.required]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sns-config",template:'
\n \n tb.rulenode.topic-arn-pattern\n \n \n {{ \'tb.rulenode.topic-arn-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),se=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.sqsQueueType=G,n.sqsQueueTypes=Object.keys(G),n.sqsQueueTypeTranslationsMap=z,n}return y(r,e),r.prototype.configForm=function(){return this.sqsConfigForm},r.prototype.onConfigurationSet=function(e){this.sqsConfigForm=this.fb.group({queueType:[e?e.queueType:null,[i.Validators.required]],queueUrlPattern:[e?e.queueUrlPattern:null,[i.Validators.required]],delaySeconds:[e?e.delaySeconds:null,[i.Validators.min(0),i.Validators.max(900)]],messageAttributes:[e?e.messageAttributes:null,[]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sqs-config",template:'
\n \n tb.rulenode.queue-type\n \n \n {{ sqsQueueTypeTranslationsMap.get(type) | translate }}\n \n \n \n \n tb.rulenode.queue-url-pattern\n \n \n {{ \'tb.rulenode.queue-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.delay-seconds\n \n \n {{ \'tb.rulenode.min-delay-seconds-message\' | translate }}\n \n \n {{ \'tb.rulenode.max-delay-seconds-message\' | translate }}\n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),me=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.pubSubConfigForm},r.prototype.onConfigurationSet=function(e){this.pubSubConfigForm=this.fb.group({projectId:[e?e.projectId:null,[i.Validators.required]],topicName:[e?e.topicName:null,[i.Validators.required]],serviceAccountKey:[e?e.serviceAccountKey:null,[i.Validators.required]],serviceAccountKeyFileName:[e?e.serviceAccountKeyFileName:null,[i.Validators.required]],messageAttributes:[e?e.messageAttributes:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-pub-sub-config",template:'
\n \n tb.rulenode.gcp-project-id\n \n \n {{ \'tb.rulenode.gcp-project-id-required\' | translate }}\n \n \n \n tb.rulenode.pubsub-topic-name\n \n \n {{ \'tb.rulenode.pubsub-topic-name-required\' | translate }}\n \n \n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ue=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.ackValues=["all","-1","0","1"],n.ToByteStandartCharsetTypesValues=W,n.ToByteStandartCharsetTypeTranslationMap=J,n}return y(r,e),r.prototype.configForm=function(){return this.kafkaConfigForm},r.prototype.onConfigurationSet=function(e){this.kafkaConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],bootstrapServers:[e?e.bootstrapServers:null,[i.Validators.required]],retries:[e?e.retries:null,[i.Validators.min(0)]],batchSize:[e?e.batchSize:null,[i.Validators.min(0)]],linger:[e?e.linger:null,[i.Validators.min(0)]],bufferMemory:[e?e.bufferMemory:null,[i.Validators.min(0)]],acks:[e?e.acks:null,[i.Validators.required]],keySerializer:[e?e.keySerializer:null,[i.Validators.required]],valueSerializer:[e?e.valueSerializer:null,[i.Validators.required]],otherProperties:[e?e.otherProperties:null,[]],addMetadataKeyValuesAsKafkaHeaders:[!!e&&e.addMetadataKeyValuesAsKafkaHeaders,[]],kafkaHeadersCharset:[e?e.kafkaHeadersCharset:null,[]]})},r.prototype.validatorTriggers=function(){return["addMetadataKeyValuesAsKafkaHeaders"]},r.prototype.updateValidators=function(e){this.kafkaConfigForm.get("addMetadataKeyValuesAsKafkaHeaders").value?this.kafkaConfigForm.get("kafkaHeadersCharset").setValidators([i.Validators.required]):this.kafkaConfigForm.get("kafkaHeadersCharset").setValidators([]),this.kafkaConfigForm.get("kafkaHeadersCharset").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-kafka-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n tb.rulenode.bootstrap-servers\n \n \n {{ \'tb.rulenode.bootstrap-servers-required\' | translate }}\n \n \n \n tb.rulenode.retries\n \n \n {{ \'tb.rulenode.min-retries-message\' | translate }}\n \n \n \n tb.rulenode.batch-size-bytes\n \n \n {{ \'tb.rulenode.min-batch-size-bytes-message\' | translate }}\n \n \n \n tb.rulenode.linger-ms\n \n \n {{ \'tb.rulenode.min-linger-ms-message\' | translate }}\n \n \n \n tb.rulenode.buffer-memory-bytes\n \n \n {{ \'tb.rulenode.min-buffer-memory-bytes-message\' | translate }}\n \n \n \n tb.rulenode.acks\n \n \n {{ ackValue }}\n \n \n \n \n tb.rulenode.key-serializer\n \n \n {{ \'tb.rulenode.key-serializer-required\' | translate }}\n \n \n \n tb.rulenode.value-serializer\n \n \n {{ \'tb.rulenode.value-serializer-required\' | translate }}\n \n \n \n \n \n \n {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}\n \n
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
\n \n tb.rulenode.charset-encoding\n \n \n {{ ToByteStandartCharsetTypeTranslationMap.get(charset) | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),de=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allMqttCredentialsTypes=$,n.mqttCredentialsTypeTranslationsMap=_,n}return y(r,e),r.prototype.configForm=function(){return this.mqttConfigForm},r.prototype.onConfigurationSet=function(e){this.mqttConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],connectTimeoutSec:[e?e.connectTimeoutSec:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(200)]],clientId:[e?e.clientId:null,[]],cleanSession:[!!e&&e.cleanSession,[]],ssl:[!!e&&e.ssl,[]],credentials:this.fb.group({type:[e&&e.credentials?e.credentials.type:null,[i.Validators.required]],username:[e&&e.credentials?e.credentials.username:null,[]],password:[e&&e.credentials?e.credentials.password:null,[]],caCert:[e&&e.credentials?e.credentials.caCert:null,[]],caCertFileName:[e&&e.credentials?e.credentials.caCertFileName:null,[]],privateKey:[e&&e.credentials?e.credentials.privateKey:null,[]],privateKeyFileName:[e&&e.credentials?e.credentials.privateKeyFileName:null,[]],cert:[e&&e.credentials?e.credentials.cert:null,[]],certFileName:[e&&e.credentials?e.credentials.certFileName:null,[]]})})},r.prototype.prepareOutputConfig=function(e){var t=e.credentials.type;switch(t){case"anonymous":e.credentials={type:t};break;case"basic":e.credentials={type:t,username:e.credentials.username,password:e.credentials.password};break;case"cert.PEM":delete e.credentials.username}return e},r.prototype.validatorTriggers=function(){return["credentials.type"]},r.prototype.updateValidators=function(e){var t=this.mqttConfigForm.get("credentials"),r=t.get("type").value;switch(e&&t.reset({type:r},{emitEvent:!1}),t.get("username").setValidators([]),t.get("password").setValidators([]),t.get("caCert").setValidators([]),t.get("caCertFileName").setValidators([]),t.get("privateKey").setValidators([]),t.get("privateKeyFileName").setValidators([]),t.get("cert").setValidators([]),t.get("certFileName").setValidators([]),r){case"anonymous":break;case"basic":t.get("username").setValidators([i.Validators.required]),t.get("password").setValidators([i.Validators.required]);break;case"cert.PEM":t.get("caCert").setValidators([i.Validators.required]),t.get("caCertFileName").setValidators([i.Validators.required]),t.get("privateKey").setValidators([i.Validators.required]),t.get("privateKeyFileName").setValidators([i.Validators.required]),t.get("cert").setValidators([i.Validators.required]),t.get("certFileName").setValidators([i.Validators.required])}t.get("username").updateValueAndValidity({emitEvent:e}),t.get("password").updateValueAndValidity({emitEvent:e}),t.get("caCert").updateValueAndValidity({emitEvent:e}),t.get("caCertFileName").updateValueAndValidity({emitEvent:e}),t.get("privateKey").updateValueAndValidity({emitEvent:e}),t.get("privateKeyFileName").updateValueAndValidity({emitEvent:e}),t.get("cert").updateValueAndValidity({emitEvent:e}),t.get("certFileName").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-mqtt-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n \n tb.rulenode.connect-timeout\n \n \n {{ \'tb.rulenode.connect-timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n
\n \n tb.rulenode.client-id\n \n \n \n {{ \'tb.rulenode.clean-session\' | translate }}\n \n \n {{ \'tb.rulenode.enable-ssl\' | translate }}\n \n \n \n tb.rulenode.credentials\n \n {{ mqttCredentialsTypeTranslationsMap.get(mqttConfigForm.get(\'credentials\').get(\'type\').value) | translate }}\n \n \n
\n \n tb.rulenode.credentials-type\n \n \n {{ mqttCredentialsTypeTranslationsMap.get(credentialsType) | translate }}\n \n \n \n {{ \'tb.rulenode.credentials-type-required\' | translate }}\n \n \n
\n \n \n \n \n tb.rulenode.username\n \n \n {{ \'tb.rulenode.username-required\' | translate }}\n \n \n \n tb.rulenode.password\n \n \n {{ \'tb.rulenode.password-required\' | translate }}\n \n \n \n \n \n \n \n \n \n \n \n tb.rulenode.private-key-password\n \n \n \n
\n
\n \n
\n',styles:[":host .tb-mqtt-credentials-panel-group{margin:0 6px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),pe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],n}return y(r,e),r.prototype.configForm=function(){return this.rabbitMqConfigForm},r.prototype.onConfigurationSet=function(e){this.rabbitMqConfigForm=this.fb.group({exchangeNamePattern:[e?e.exchangeNamePattern:null,[]],routingKeyPattern:[e?e.routingKeyPattern:null,[]],messageProperties:[e?e.messageProperties:null,[]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],virtualHost:[e?e.virtualHost:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]],automaticRecoveryEnabled:[!!e&&e.automaticRecoveryEnabled,[]],connectionTimeout:[e?e.connectionTimeout:null,[i.Validators.min(0)]],handshakeTimeout:[e?e.handshakeTimeout:null,[i.Validators.min(0)]],clientProperties:[e?e.clientProperties:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rabbit-mq-config",template:'
\n \n tb.rulenode.exchange-name-pattern\n \n \n \n tb.rulenode.routing-key-pattern\n \n \n \n tb.rulenode.message-properties\n \n \n {{ property }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n
\n \n tb.rulenode.virtual-host\n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n \n {{ \'tb.rulenode.automatic-recovery\' | translate }}\n \n \n tb.rulenode.connection-timeout-ms\n \n \n {{ \'tb.rulenode.min-connection-timeout-ms-message\' | translate }}\n \n \n \n tb.rulenode.handshake-timeout-ms\n \n \n {{ \'tb.rulenode.min-handshake-timeout-ms-message\' | translate }}\n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ce=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.httpRequestTypes=Object.keys(Q),n}return y(r,e),r.prototype.configForm=function(){return this.restApiCallConfigForm},r.prototype.onConfigurationSet=function(e){this.restApiCallConfigForm=this.fb.group({restEndpointUrlPattern:[e?e.restEndpointUrlPattern:null,[i.Validators.required]],requestMethod:[e?e.requestMethod:null,[i.Validators.required]],useSimpleClientHttpFactory:[!!e&&e.useSimpleClientHttpFactory,[]],readTimeoutMs:[e?e.readTimeoutMs:null,[]],maxParallelRequestsCount:[e?e.maxParallelRequestsCount:null,[i.Validators.min(0)]],headers:[e?e.headers:null,[]],useRedisQueueForMsgPersistence:[!!e&&e.useRedisQueueForMsgPersistence,[]],trimQueue:[!!e&&e.trimQueue,[]],maxQueueSize:[e?e.maxQueueSize:null,[]]})},r.prototype.validatorTriggers=function(){return["useSimpleClientHttpFactory","useRedisQueueForMsgPersistence"]},r.prototype.updateValidators=function(e){var t=this.restApiCallConfigForm.get("useSimpleClientHttpFactory").value,r=this.restApiCallConfigForm.get("useRedisQueueForMsgPersistence").value;t?this.restApiCallConfigForm.get("readTimeoutMs").setValidators([]):this.restApiCallConfigForm.get("readTimeoutMs").setValidators([i.Validators.min(0)]),r?this.restApiCallConfigForm.get("maxQueueSize").setValidators([i.Validators.min(0)]):this.restApiCallConfigForm.get("maxQueueSize").setValidators([]),this.restApiCallConfigForm.get("readTimeoutMs").updateValueAndValidity({emitEvent:e}),this.restApiCallConfigForm.get("maxQueueSize").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rest-api-call-config",template:'
\n \n tb.rulenode.endpoint-url-pattern\n \n \n {{ \'tb.rulenode.endpoint-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.request-method\n \n \n {{ requestType }}\n \n \n \n \n {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}\n \n \n tb.rulenode.read-timeout\n \n \n \n \n tb.rulenode.max-parallel-requests-count\n \n \n \n \n
tb.rulenode.headers-hint
\n \n \n \n {{ \'tb.rulenode.use-redis-queue\' | translate }}\n \n
\n \n {{ \'tb.rulenode.trim-redis-queue\' | translate }}\n \n \n tb.rulenode.redis-queue-max-size\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),fe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.smtpProtocols=["smtp","smtps"],n.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],n}return y(r,e),r.prototype.configForm=function(){return this.sendEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.sendEmailConfigForm=this.fb.group({useSystemSmtpSettings:[!!e&&e.useSystemSmtpSettings,[]],smtpProtocol:[e?e.smtpProtocol:null,[]],smtpHost:[e?e.smtpHost:null,[]],smtpPort:[e?e.smtpPort:null,[]],timeout:[e?e.timeout:null,[]],enableTls:[!!e&&e.enableTls,[]],tlsVersion:[e?e.tlsVersion:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]]})},r.prototype.validatorTriggers=function(){return["useSystemSmtpSettings"]},r.prototype.updateValidators=function(e){this.sendEmailConfigForm.get("useSystemSmtpSettings").value?(this.sendEmailConfigForm.get("smtpProtocol").setValidators([]),this.sendEmailConfigForm.get("smtpHost").setValidators([]),this.sendEmailConfigForm.get("smtpPort").setValidators([]),this.sendEmailConfigForm.get("timeout").setValidators([])):(this.sendEmailConfigForm.get("smtpProtocol").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpHost").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpPort").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]),this.sendEmailConfigForm.get("timeout").setValidators([i.Validators.required,i.Validators.min(0)])),this.sendEmailConfigForm.get("smtpProtocol").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpHost").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpPort").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("timeout").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-send-email-config",template:'
\n \n {{ \'tb.rulenode.use-system-smtp-settings\' | translate }}\n \n
\n \n tb.rulenode.smtp-protocol\n \n \n {{ smtpProtocol.toUpperCase() }}\n \n \n \n
\n \n tb.rulenode.smtp-host\n \n \n {{ \'tb.rulenode.smtp-host-required\' | translate }}\n \n \n \n tb.rulenode.smtp-port\n \n \n {{ \'tb.rulenode.smtp-port-required\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n
\n \n tb.rulenode.timeout-msec\n \n \n {{ \'tb.rulenode.timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-timeout-msec-message\' | translate }}\n \n \n \n {{ \'tb.rulenode.enable-tls\' | translate }}\n \n \n tb.rulenode.tls-version\n \n \n {{ tlsVersion }}\n \n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ge=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.serviceType=a.ServiceType.TB_RULE_ENGINE,n}return y(r,e),r.prototype.configForm=function(){return this.checkPointConfigForm},r.prototype.onConfigurationSet=function(e){this.checkPointConfigForm=this.fb.group({queueName:[e?e.queueName:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-check-point-config",template:'
\n \n \n
tb.rulenode.select-queue-hint
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ye=function(){function e(){}return e=b([t.NgModule({declarations:[T,x,q,S,I,k,N,V,E,A,L,Y,Z,X,ee,ie,le,se,me,ue,de,pe,ce,fe,ge],imports:[r.CommonModule,a.SharedModule,oe],exports:[T,x,q,S,I,k,N,V,E,A,L,Y,Z,X,ee,ie,le,se,me,ue,de,pe,ce,fe,ge]})],e)}(),be=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.checkMessageConfigForm},r.prototype.onConfigurationSet=function(e){this.checkMessageConfigForm=this.fb.group({messageNames:[e?e.messageNames:null,[]],metadataNames:[e?e.metadataNames:null,[]],checkAllKeys:[!!e&&e.checkAllKeys,[]]})},r.prototype.validateConfig=function(){var e=this.checkMessageConfigForm.get("messageNames").value,t=this.checkMessageConfigForm.get("metadataNames").value;return e.length>0||t.length>0},r.prototype.removeMessageName=function(e){var t=this.checkMessageConfigForm.get("messageNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("messageNames").setValue(t,{emitEvent:!0}))},r.prototype.removeMetadataName=function(e){var t=this.checkMessageConfigForm.get("metadataNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("metadataNames").setValue(t,{emitEvent:!0}))},r.prototype.addMessageName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("messageNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("messageNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.prototype.addMetadataName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("metadataNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("metadataNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-message-config",template:'
\n \n \n \n \n \n {{messageName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n \n \n \n \n {{metadataName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n {{ \'tb.rulenode.check-all-keys\' | translate }}\n \n
tb.rulenode.check-all-keys-hint
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),he=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.entitySearchDirection=Object.keys(a.EntitySearchDirection),n.entitySearchDirectionTranslationsMap=a.entitySearchDirectionTranslations,n}return y(r,e),r.prototype.configForm=function(){return this.checkRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.checkRelationConfigForm=this.fb.group({checkForSingleEntity:[!!e&&e.checkForSingleEntity,[]],direction:[e?e.direction:null,[]],entityType:[e?e.entityType:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],entityId:[e?e.entityId:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],relationType:[e?e.relationType:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["checkForSingleEntity"]},r.prototype.updateValidators=function(e){var t=this.checkRelationConfigForm.get("checkForSingleEntity").value;this.checkRelationConfigForm.get("entityType").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:e}),this.checkRelationConfigForm.get("entityId").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityId").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-relation-config",template:'
\n \n {{ \'tb.rulenode.check-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.check-relation-hint
\n \n relation.direction\n \n \n {{ entitySearchDirectionTranslationsMap.get(direction) | translate }}\n \n \n \n
\n \n \n \n \n
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ce=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(O),n.rangeUnitTranslationMap=B,n}return y(r,e),r.prototype.configForm=function(){return this.geoFilterConfigForm},r.prototype.onConfigurationSet=function(e){this.geoFilterConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoFilterConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoFilterConfigForm.get("perimeterType").value;t?this.geoFilterConfigForm.get("perimeterType").setValidators([]):this.geoFilterConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoFilterConfigForm.get("centerLatitude").setValidators([]),this.geoFilterConfigForm.get("centerLongitude").setValidators([]),this.geoFilterConfigForm.get("range").setValidators([]),this.geoFilterConfigForm.get("rangeUnit").setValidators([])):(this.geoFilterConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoFilterConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoFilterConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoFilterConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoFilterConfigForm.get("polygonsDefinition").setValidators([]):this.geoFilterConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoFilterConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoFilterConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ve=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.messageTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.messageTypeConfigForm=this.fb.group({messageTypes:[e?e.messageTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-message-type-config",template:'
\n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Fe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allowedEntityTypes=[a.EntityType.DEVICE,a.EntityType.ASSET,a.EntityType.ENTITY_VIEW,a.EntityType.TENANT,a.EntityType.CUSTOMER,a.EntityType.USER,a.EntityType.DASHBOARD,a.EntityType.RULE_CHAIN,a.EntityType.RULE_NODE],n}return y(r,e),r.prototype.configForm=function(){return this.originatorTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorTypeConfigForm=this.fb.group({originatorTypes:[e?e.originatorTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-originator-type-config",template:'
\n \n \n \n
\n',styles:[":host ::ng-deep tb-entity-type-list .mat-form-field-flex{padding-top:0}:host ::ng-deep tb-entity-type-list .mat-form-field-infix{border-top:0}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Te=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"filter",this.translate.instant("tb.rulenode.filter"),"Filter",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),xe=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.switchConfigForm},r.prototype.onConfigurationSet=function(e){this.switchConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.switchConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"switch",this.translate.instant("tb.rulenode.switch"),"Switch",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.switchConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-switch-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),qe=function(e){function r(t,r,n){var o,l,s=e.call(this,t)||this;s.store=t,s.translate=r,s.fb=n,s.alarmStatusTranslationsMap=a.alarmStatusTranslations,s.alarmStatusList=[],s.searchText="",s.displayStatusFn=s.displayStatus.bind(s);try{for(var m=C(Object.keys(a.AlarmStatus)),u=m.next();!u.done;u=m.next()){var d=u.value;s.alarmStatusList.push(a.AlarmStatus[d])}}catch(e){o={error:e}}finally{try{u&&!u.done&&(l=m.return)&&l.call(m)}finally{if(o)throw o.error}}return s.statusFormControl=new i.FormControl(""),s.filteredAlarmStatus=s.statusFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return s.fetchAlarmStatus(e)})),f.share()),s}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.alarmStatusConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.statusFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.alarmStatusConfigForm=this.fb.group({alarmStatusList:[e?e.alarmStatusList:null,[i.Validators.required]]})},r.prototype.displayStatus=function(e){return e?this.translate.instant(a.alarmStatusTranslations.get(e)):void 0},r.prototype.fetchAlarmStatus=function(e){var t=this,r=this.getAlarmStatusList();if(this.searchText=e,this.searchText&&this.searchText.length){var n=this.searchText.toUpperCase();return c.of(r.filter((function(e){return t.translate.instant(a.alarmStatusTranslations.get(a.AlarmStatus[e])).toUpperCase().includes(n)})))}return c.of(r)},r.prototype.alarmStatusSelected=function(e){this.addAlarmStatus(e.option.value),this.clear("")},r.prototype.removeAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))}},r.prototype.addAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))},r.prototype.getAlarmStatusList=function(){var e=this;return this.alarmStatusList.filter((function(t){return-1===e.alarmStatusConfigForm.get("alarmStatusList").value.indexOf(t)}))},r.prototype.onAlarmStatusInputFocus=function(){this.statusFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.alarmStatusInput.nativeElement.value=e,this.statusFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.alarmStatusInput.nativeElement.blur(),t.alarmStatusInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("alarmStatusInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"alarmStatusInput",void 0),r=b([t.Component({selector:"tb-filter-node-check-alarm-status-config",template:'
\n \n tb.rulenode.alarm-status-filter\n \n \n \n {{alarmStatusTranslationsMap.get(alarmStatus) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-alarm-status-matching\n
\n
\n \n \n \n \n
\n\n\n\n'}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Se=function(){function e(){}return e=b([t.NgModule({declarations:[be,he,Ce,ve,Fe,Te,xe,qe],imports:[r.CommonModule,a.SharedModule,oe],exports:[be,he,Ce,ve,Fe,Te,xe,qe]})],e)}(),Ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.customerAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.customerAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-customer-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ke=function(e){function r(t,r,n){var a,o,l=e.call(this,t)||this;l.store=t,l.translate=r,l.fb=n,l.entityDetailsTranslationsMap=H,l.entityDetailsList=[],l.searchText="",l.displayDetailsFn=l.displayDetails.bind(l);try{for(var s=C(Object.keys(K)),m=s.next();!m.done;m=s.next()){var u=m.value;l.entityDetailsList.push(K[u])}}catch(e){a={error:e}}finally{try{m&&!m.done&&(o=s.return)&&o.call(s)}finally{if(a)throw a.error}}return l.detailsFormControl=new i.FormControl(""),l.filteredEntityDetails=l.detailsFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return l.fetchEntityDetails(e)})),f.share()),l}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.entityDetailsConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.detailsFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.entityDetailsConfigForm=this.fb.group({detailsList:[e?e.detailsList:null,[i.Validators.required]],addToMetadata:[!!e&&e.addToMetadata,[]]})},r.prototype.displayDetails=function(e){return e?this.translate.instant(H.get(e)):void 0},r.prototype.fetchEntityDetails=function(e){var t=this;if(this.searchText=e,this.searchText&&this.searchText.length){var r=this.searchText.toUpperCase();return c.of(this.entityDetailsList.filter((function(e){return t.translate.instant(H.get(K[e])).toUpperCase().includes(r)})))}return c.of(this.entityDetailsList)},r.prototype.detailsFieldSelected=function(e){this.addDetailsField(e.option.value),this.clear("")},r.prototype.removeDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.entityDetailsConfigForm.get("detailsList").setValue(t))}},r.prototype.addDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.entityDetailsConfigForm.get("detailsList").setValue(t))},r.prototype.onEntityDetailsInputFocus=function(){this.detailsFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.detailsInput.nativeElement.value=e,this.detailsFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.detailsInput.nativeElement.blur(),t.detailsInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("detailsInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"detailsInput",void 0),r=b([t.Component({selector:"tb-enrichment-node-entity-details-config",template:'
\n \n tb.rulenode.entity-details\n \n \n \n {{entityDetailsTranslationsMap.get(details) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-entity-details-matching\n
\n
\n \n \n \n \n \n {{ \'tb.rulenode.add-to-metadata\' | translate }}\n \n
tb.rulenode.add-to-metadata-hint
\n
\n',styles:[":host ::ng-deep mat-form-field.entity-fields-list .mat-form-field-wrapper{margin-bottom:-1.25em}"]}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ne=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.deviceAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.deviceAttributesConfigForm=this.fb.group({deviceRelationsQuery:[e?e.deviceRelationsQuery:null,[i.Validators.required]],tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.deviceAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.deviceAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.deviceAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.deviceAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-device-attributes-config",template:'
\n \n \n \n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ve=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.originatorAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorAttributesConfigForm=this.fb.group({tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.originatorAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.originatorAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.originatorAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.originatorAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-attributes-config",template:'
\n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.originatorFieldsConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorFieldsConfigForm=this.fb.group({fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-fields-config",template:'
\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ae=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n.fetchMode=j,n.fetchModes=Object.keys(j),n.samplingOrders=Object.keys(U),n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=D,n}return y(r,e),r.prototype.configForm=function(){return this.getTelemetryFromDatabaseConfigForm},r.prototype.onConfigurationSet=function(e){this.getTelemetryFromDatabaseConfigForm=this.fb.group({latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],fetchMode:[e?e.fetchMode:null,[i.Validators.required]],orderBy:[e?e.orderBy:null,[]],limit:[e?e.limit:null,[]],useMetadataIntervalPatterns:[!!e&&e.useMetadataIntervalPatterns,[]],startInterval:[e?e.startInterval:null,[]],startIntervalTimeUnit:[e?e.startIntervalTimeUnit:null,[]],endInterval:[e?e.endInterval:null,[]],endIntervalTimeUnit:[e?e.endIntervalTimeUnit:null,[]],startIntervalPattern:[e?e.startIntervalPattern:null,[]],endIntervalPattern:[e?e.endIntervalPattern:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchMode","useMetadataIntervalPatterns"]},r.prototype.updateValidators=function(e){var t=this.getTelemetryFromDatabaseConfigForm.get("fetchMode").value,r=this.getTelemetryFromDatabaseConfigForm.get("useMetadataIntervalPatterns").value;t&&t===j.ALL?(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([i.Validators.required,i.Validators.min(2),i.Validators.max(1e3)])):(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([])),r?(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([i.Validators.required])):(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([])),this.getTelemetryFromDatabaseConfigForm.get("orderBy").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("limit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").updateValueAndValidity({emitEvent:e})},r.prototype.removeKey=function(e,t){var r=this.getTelemetryFromDatabaseConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.getTelemetryFromDatabaseConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-get-telemetry-from-database",template:'
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n tb.rulenode.fetch-mode\n \n \n {{ mode }}\n \n \n tb.rulenode.fetch-mode-hint\n \n
\n \n tb.rulenode.order-by\n \n \n {{ order }}\n \n \n tb.rulenode.order-by-hint\n \n \n tb.rulenode.limit\n \n tb.rulenode.limit-hint\n \n
\n \n {{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-interval-patterns-hint
\n
\n
\n \n tb.rulenode.start-interval\n \n \n {{ \'tb.rulenode.start-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.start-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.end-interval\n \n \n {{ \'tb.rulenode.end-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.end-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n \n tb.rulenode.start-interval-pattern\n \n \n {{ \'tb.rulenode.start-interval-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.end-interval-pattern\n \n \n {{ \'tb.rulenode.end-interval-pattern-required\' | translate }}\n \n \n \n \n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.relatedAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.relatedAttributesConfigForm=this.fb.group({relationsQuery:[e?e.relationsQuery:null,[i.Validators.required]],telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-related-attributes-config",template:'
\n \n \n \n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Me=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.tenantAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.tenantAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-tenant-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Pe=function(){function e(){}return e=b([t.NgModule({declarations:[Ie,ke,Ne,Ve,Ee,Ae,Le,Me],imports:[r.CommonModule,a.SharedModule,oe],exports:[Ie,ke,Ne,Ve,Ee,Ae,Le,Me]})],e)}(),Re=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.originatorSource=v,n.originatorSources=Object.keys(v),n.originatorSourceTranslationMap=P,n}return y(r,e),r.prototype.configForm=function(){return this.changeOriginatorConfigForm},r.prototype.onConfigurationSet=function(e){this.changeOriginatorConfigForm=this.fb.group({originatorSource:[e?e.originatorSource:null,[i.Validators.required]],relationsQuery:[e?e.relationsQuery:null,[]]})},r.prototype.validatorTriggers=function(){return["originatorSource"]},r.prototype.updateValidators=function(e){var t=this.changeOriginatorConfigForm.get("originatorSource").value;t&&t===v.RELATED?this.changeOriginatorConfigForm.get("relationsQuery").setValidators([i.Validators.required]):this.changeOriginatorConfigForm.get("relationsQuery").setValidators([]),this.changeOriginatorConfigForm.get("relationsQuery").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-change-originator-config",template:'
\n \n tb.rulenode.originator-source\n \n \n {{ originatorSourceTranslationMap.get(source) | translate }}\n \n \n \n
\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),we=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"update",this.translate.instant("tb.rulenode.transformer"),"Transform",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-transformation-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),Oe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.toEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.toEmailConfigForm=this.fb.group({fromTemplate:[e?e.fromTemplate:null,[i.Validators.required]],toTemplate:[e?e.toTemplate:null,[i.Validators.required]],ccTemplate:[e?e.ccTemplate:null,[]],bccTemplate:[e?e.bccTemplate:null,[]],subjectTemplate:[e?e.subjectTemplate:null,[i.Validators.required]],bodyTemplate:[e?e.bodyTemplate:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-to-email-config",template:'
\n \n tb.rulenode.from-template\n \n \n {{ \'tb.rulenode.from-template-required\' | translate }}\n \n \n \n \n tb.rulenode.to-template\n \n \n {{ \'tb.rulenode.to-template-required\' | translate }}\n \n \n \n \n tb.rulenode.cc-template\n \n \n \n \n tb.rulenode.bcc-template\n \n \n \n \n tb.rulenode.subject-template\n \n \n {{ \'tb.rulenode.subject-template-required\' | translate }}\n \n \n \n \n tb.rulenode.body-template\n \n \n {{ \'tb.rulenode.body-template-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),De=function(){function e(){}return e=b([t.NgModule({declarations:[Re,we,Oe],imports:[r.CommonModule,a.SharedModule,oe],exports:[Re,we,Oe]})],e)}(),Ke=function(){function e(e){!function(e){e.setTranslation("en_US",{tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-status-filter":"Alarm status filter","alarm-status-list-empty":"Alarm status list is empty","no-alarm-status-matching":"No alarm status matching were found.",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}},!0)}(e)}return e.ctorParameters=function(){return[{type:n.TranslateService}]},e=b([t.NgModule({declarations:[F],imports:[r.CommonModule,a.SharedModule],exports:[ye,Se,Pe,De,F]}),h("design:paramtypes",[n.TranslateService])],e)}();e.RuleNodeCoreConfigModule=Ke,e.ɵa=F,e.ɵb=ye,e.ɵba=ge,e.ɵbb=oe,e.ɵbc=te,e.ɵbd=re,e.ɵbe=ne,e.ɵbf=ae,e.ɵbg=Se,e.ɵbh=be,e.ɵbi=he,e.ɵbj=Ce,e.ɵbk=ve,e.ɵbl=Fe,e.ɵbm=Te,e.ɵbn=xe,e.ɵbo=qe,e.ɵbp=Pe,e.ɵbq=Ie,e.ɵbr=ke,e.ɵbs=Ne,e.ɵbt=Ve,e.ɵbu=Ee,e.ɵbv=Ae,e.ɵbw=Le,e.ɵbx=Me,e.ɵby=De,e.ɵbz=Re,e.ɵc=T,e.ɵca=we,e.ɵcb=Oe,e.ɵd=x,e.ɵe=q,e.ɵf=S,e.ɵg=I,e.ɵh=k,e.ɵi=N,e.ɵj=V,e.ɵk=E,e.ɵl=A,e.ɵm=L,e.ɵn=Y,e.ɵo=Z,e.ɵp=X,e.ɵq=ee,e.ɵr=ie,e.ɵs=le,e.ɵt=se,e.ɵu=me,e.ɵv=ue,e.ɵw=de,e.ɵx=pe,e.ɵy=ce,e.ɵz=fe,Object.defineProperty(e,"__esModule",{value:!0})})); //# sourceMappingURL=rulenode-core-config.umd.min.js.map \ No newline at end of file diff --git a/ui-ngx/package-lock.json b/ui-ngx/package-lock.json index aec8d12a34..3232926e05 100644 --- a/ui-ngx/package-lock.json +++ b/ui-ngx/package-lock.json @@ -1816,7 +1816,8 @@ "@types/geojson": { "version": "7946.0.7", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.7.tgz", - "integrity": "sha512-wE2v81i4C4Ol09RtsWFAqg3BUitWbHSpSlIo+bNdsCJijO9sjme+zm+73ZMCa/qMC8UEERxzGbvmr1cffo2SiQ==" + "integrity": "sha512-wE2v81i4C4Ol09RtsWFAqg3BUitWbHSpSlIo+bNdsCJijO9sjme+zm+73ZMCa/qMC8UEERxzGbvmr1cffo2SiQ==", + "dev": true }, "@types/glob": { "version": "7.1.1", @@ -1880,6 +1881,7 @@ "version": "1.5.12", "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.5.12.tgz", "integrity": "sha512-61HRMIng+bWvnnAIqUWLBlrd/TQZc4gU+gN1JL4K47EDtwIrcMEhWgi7PdcpbG1YmpH4F0EfOimkvV82gJIl9w==", + "dev": true, "requires": { "@types/geojson": "*" } @@ -1897,6 +1899,7 @@ "version": "1.4.2", "resolved": "https://registry.npmjs.org/@types/leaflet.markercluster/-/leaflet.markercluster-1.4.2.tgz", "integrity": "sha512-QQ//hevAxMH2dlRQdRre7V/1G+TbtuDtZnZF/75TNwVIgklrsQVCIcS/cvLsl7UUryfPJ6xmoYHfFzK5iGVgpg==", + "dev": true, "requires": { "@types/leaflet": "*" } @@ -8611,7 +8614,7 @@ "integrity": "sha512-4O3GWAYJaauMCILm07weko2rHA8a4kjn7+8Lg4s1d7SxwS/3IpkVD/GljbRrIJ1c1W/XGJ3GbuK7RyYZEJChhw==" }, "ngx-flowchart": { - "version": "git://github.com/thingsboard/ngx-flowchart.git#97a77477ca8579becf0e3a07866046b4536fe30a", + "version": "git://github.com/thingsboard/ngx-flowchart.git#a4157b0eef2eb3646ef920447c7b06b39d54f87f", "from": "git://github.com/thingsboard/ngx-flowchart.git#master", "requires": { "tslib": "^1.10.0" From 916579cf05aaa58b1d4931857501866b74d9eef9 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 24 Apr 2020 17:21:29 +0300 Subject: [PATCH 091/102] Fix bundles budget --- ui-ngx/package-lock.json | 375 +++++++++++------- .../components/widget/lib/maps/map-widget2.ts | 2 +- 2 files changed, 224 insertions(+), 153 deletions(-) diff --git a/ui-ngx/package-lock.json b/ui-ngx/package-lock.json index 3232926e05..611439194e 100644 --- a/ui-ngx/package-lock.json +++ b/ui-ngx/package-lock.json @@ -19,12 +19,12 @@ } }, "@angular-devkit/architect": { - "version": "0.901.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.901.1.tgz", - "integrity": "sha512-foWDAurMfBDYLAJxHpTFkJBub1c2A8+eWHbBjgqIHmT8xadnE7t8nSA9XDl+k/kIoWw/qFU+6IttPirudYc/vw==", + "version": "0.901.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.901.3.tgz", + "integrity": "sha512-CFjSj48nOJwejmFFtenIqSZWyxRe4fRQsg16l0R4sagW7YwMJSaW6Yl9hRHM8bviPRrTpGHnxeq1x506v1ARLw==", "dev": true, "requires": { - "@angular-devkit/core": "9.1.1", + "@angular-devkit/core": "9.1.3", "rxjs": "6.5.4" }, "dependencies": { @@ -40,21 +40,21 @@ } }, "@angular-devkit/build-angular": { - "version": "0.901.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-0.901.1.tgz", - "integrity": "sha512-6uEvo5htsJoxQHBVwHOGmM6YWq5q6m9UWMv/ughlek0RtSLFfOt9TZQ/yQHgtGQsCQvscD/jBzVoD0zD5Ax/SQ==", + "version": "0.901.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-0.901.3.tgz", + "integrity": "sha512-0ivl+rxiFWY/cWpAhflaJfUmTGNRZn3m4gyt0YNFyHh7oJcwKFVpd9lxJAM4t47Xfyj0ICpTL99UBi4aeo3lEg==", "dev": true, "requires": { - "@angular-devkit/architect": "0.901.1", - "@angular-devkit/build-optimizer": "0.901.1", - "@angular-devkit/build-webpack": "0.901.1", - "@angular-devkit/core": "9.1.1", + "@angular-devkit/architect": "0.901.3", + "@angular-devkit/build-optimizer": "0.901.3", + "@angular-devkit/build-webpack": "0.901.3", + "@angular-devkit/core": "9.1.3", "@babel/core": "7.9.0", "@babel/generator": "7.9.3", "@babel/preset-env": "7.9.0", "@babel/template": "7.8.6", "@jsdevtools/coverage-istanbul-loader": "3.0.3", - "@ngtools/webpack": "9.1.1", + "@ngtools/webpack": "9.1.3", "ajv": "6.12.0", "autoprefixer": "9.7.4", "babel-loader": "8.0.6", @@ -64,6 +64,7 @@ "circular-dependency-plugin": "5.2.0", "copy-webpack-plugin": "5.1.1", "core-js": "3.6.4", + "css-loader": "3.5.1", "cssnano": "4.1.10", "file-loader": "6.0.0", "find-cache-dir": "3.3.1", @@ -91,7 +92,6 @@ "semver": "7.1.3", "source-map": "0.7.3", "source-map-loader": "0.2.4", - "source-map-support": "0.5.16", "speed-measure-webpack-plugin": "1.3.1", "style-loader": "1.1.3", "stylus": "0.54.7", @@ -138,9 +138,9 @@ } }, "@angular-devkit/build-optimizer": { - "version": "0.901.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/build-optimizer/-/build-optimizer-0.901.1.tgz", - "integrity": "sha512-o0A9CcyDQSUnC5CQIKf92VH8amIYRYrMgLf2kdhSMcy0QV+rEJyN81dSvwX/Yxgnr9NbWEAQg7jnyKk06vfhOw==", + "version": "0.901.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/build-optimizer/-/build-optimizer-0.901.3.tgz", + "integrity": "sha512-fLRiQ1egH1qsmj5GMJ4ZWawtUYLwpIz1hKJ3WwAIFEFJF2+CDWhO1feUaXZRxPf5qwoj8S+8cSWjdcnKLf/mBw==", "dev": true, "requires": { "loader-utils": "2.0.0", @@ -159,13 +159,13 @@ } }, "@angular-devkit/build-webpack": { - "version": "0.901.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.901.1.tgz", - "integrity": "sha512-9oNI+wPSk8yECy+f0EebfMx4PH3uDJRrifYZAxcr84IpzEbpfpRuYhE3ecwqd7k0zu2Kdjw1uUrGxBuH4/sbGg==", + "version": "0.901.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.901.3.tgz", + "integrity": "sha512-tZOiVNCrKaEAo7SuyXPpOoaPy/pbX70G8NFtyhKvoGt5dPbgybWTlibACyzOciBI9wUlGSaJiGPZzKTgL4O5OA==", "dev": true, "requires": { - "@angular-devkit/architect": "0.901.1", - "@angular-devkit/core": "9.1.1", + "@angular-devkit/architect": "0.901.3", + "@angular-devkit/core": "9.1.3", "rxjs": "6.5.4" }, "dependencies": { @@ -181,9 +181,9 @@ } }, "@angular-devkit/core": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-9.1.1.tgz", - "integrity": "sha512-57MNew2u1QwVb69jxZyhXgdW9kqcGyWyRy2ui/hWCkWLg7RumWtyypmdTs89FNExB4HqtXlQ2eO3JZxfs7QR3w==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-9.1.3.tgz", + "integrity": "sha512-VRV96prPy0Kdlm6XmI7DITqSMSc1bINimnOhzQre3euDX5OQty+EUqaexHtMv/SPDZX1agP+buHr6viv9YEhzA==", "dev": true, "requires": { "ajv": "6.12.0", @@ -205,12 +205,12 @@ } }, "@angular-devkit/schematics": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-9.1.1.tgz", - "integrity": "sha512-6wx2HcvafHvEjEa1tjDzW2hXrOiSE8ALqJUArb3+NoO1BDM42aGcqyPo0ODzKtDk12CgSsFXdNKRpQ5AmpSPtw==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-9.1.3.tgz", + "integrity": "sha512-nNRoQPqoin5jzLKmTabIPLSqVw2Zjnk78XC4AhAM7rSisWMsZ6pw5+BNDITiKfq0LkSXfOs0DoJV/Vr9mLkn/A==", "dev": true, "requires": { - "@angular-devkit/core": "9.1.1", + "@angular-devkit/core": "9.1.3", "ora": "4.0.3", "rxjs": "6.5.4" }, @@ -227,9 +227,9 @@ } }, "@angular/animations": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-9.1.2.tgz", - "integrity": "sha512-5UJ8SzCtFj4vZChVsni4K9oa4qE9tQ67bwnP6DKxkLEJKQWWyasYp+2siAi/7zD2ro2XA0qRMYhgQz5Vj6eBoQ==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-9.1.3.tgz", + "integrity": "sha512-QTQZSnXxr9SCgrN2SZsTMzS/UldrZ65y6Kuqs9yyghNc694eMbabgiPdeegjvcKlDhVPCN2x9Bjfb1vXPOfdvQ==" }, "@angular/cdk": { "version": "9.2.1", @@ -240,16 +240,16 @@ } }, "@angular/cli": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-9.1.1.tgz", - "integrity": "sha512-sjRAV4UF8M5v+2gw+EwCYSgciBZDc05AbNxQt+uUdxdfR1QU9hifWq8WDxfOR6jdDP5YqMtQsNaFNwrUyjJJoQ==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-9.1.3.tgz", + "integrity": "sha512-/7yHOuiyMgpcoBuADPrF4Eo9VDysA57fsyrMlOH2WZmKdsjW032StS9EIOue5RNQ7y0DwrtgtnkjbpZ6nYo3Pw==", "dev": true, "requires": { - "@angular-devkit/architect": "0.901.1", - "@angular-devkit/core": "9.1.1", - "@angular-devkit/schematics": "9.1.1", - "@schematics/angular": "9.1.1", - "@schematics/update": "0.901.1", + "@angular-devkit/architect": "0.901.3", + "@angular-devkit/core": "9.1.3", + "@angular-devkit/schematics": "9.1.3", + "@schematics/angular": "9.1.3", + "@schematics/update": "0.901.3", "@yarnpkg/lockfile": "1.1.0", "ansi-colors": "4.1.1", "debug": "4.1.1", @@ -288,19 +288,19 @@ } }, "@angular/common": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/common/-/common-9.1.2.tgz", - "integrity": "sha512-MAQW0DGq2NhvJSETLTwuImdzwI+wboG+5Uzgc0L8C/hX7SrEE65Hmz4nIjmGi2CPGLpodfWEcCPlV0R0dHun4A==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/common/-/common-9.1.3.tgz", + "integrity": "sha512-OGrd/x+WoceskeJWZI3ur+KqVoBHn2kmbg57RyJ8Ng/BkdDVYiXzQ/UvvX1WYGH/YzmiiVdlnVq55xkHkzGaXw==" }, "@angular/compiler": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-9.1.2.tgz", - "integrity": "sha512-82DitvlZA5DoE++qiiJdgx6HE71h96sfLHCjjvYJHApoOcnX+zBZFBrflpbXK9W4YdvydU4Lc1Ql1eleQPeT/g==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-9.1.3.tgz", + "integrity": "sha512-NTqGZUwwdm/RIG1rWPIqQwNyyRcPOmNlv+t1KpydFtycpTut0lAh0shiu57h83Cl2LgaFV4veyiAAtNmxFmqXA==" }, "@angular/compiler-cli": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-9.1.2.tgz", - "integrity": "sha512-zC/oIuT68vtuiYrgXqWsDNi/0DQ1Ti6J6gMTrXVvZXlEDikEExTAJKrrBV5jo6B0bpUofe/dDcJaKR3Ys3cM3Q==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-9.1.3.tgz", + "integrity": "sha512-FqDPmEbJQmzzxqF08CJfqT9XKJFPYdyZuFFStKkJEZ5s1JrjrBlgQSqdF0qp5jWNsjTB87USLHC5zAlzJtMKxQ==", "dev": true, "requires": { "canonical-path": "1.0.0", @@ -501,9 +501,9 @@ } }, "@angular/core": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/core/-/core-9.1.2.tgz", - "integrity": "sha512-nAqRl+5drAnZlBT7AgSo2JJubmPNrGCvhZ83dosPEUqJoLr69/lYipnF/iqKzYn5xo4MM1O27WIhSNDtDjBmzQ==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/core/-/core-9.1.3.tgz", + "integrity": "sha512-r+GyeCuBw9SUBzpPWhKxzTOkNYotYp3gANSWndihLC9P6W5T8Hfyg8uYcxKy7L42bpx8eXZOXFq91cioVwZtqA==" }, "@angular/flex-layout": { "version": "9.0.0-beta.29", @@ -511,14 +511,14 @@ "integrity": "sha512-93sxR+kYfYMOdnlWL0Q77FZ428gg8XnBu0YZm6GsCdkw/vLggIT/G1ZAqHlCPIODt6pxmCJ5KXh4ShvniIYDsA==" }, "@angular/forms": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-9.1.2.tgz", - "integrity": "sha512-/f2WhMiKDo1/RoisTI71Dy4Z4+sAsAuzgIxJEXvgDGSYvzLl9G8erFx4N6be8Cy/xXwErmp3JOwXIAXjzxF8tA==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-9.1.3.tgz", + "integrity": "sha512-t3BexN5EHgCK6q0R04AuyNb6ZUP6sd9fJ+ER98yWws3HXV6FT+J5IB7dVnhelLoU1GTnql4TJSS+LTVradKZ7g==" }, "@angular/language-service": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/language-service/-/language-service-9.1.2.tgz", - "integrity": "sha512-0BnDIFbIAtFKS/2HDI0efcnW3DSkAAFhUWluBnssvwGzuMHikOKGeamuWM27ki3IzPQKGJ7fGS2fz8A7sMeIYQ==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/language-service/-/language-service-9.1.3.tgz", + "integrity": "sha512-mcZDO6C2BpDinMjDeMAdohTefpFxNu/S0JEqewYQIQ8TXtKm77xDPUdNdNb4qD4zYW+c46f7UXNMX/tbYCicQw==", "dev": true }, "@angular/material": { @@ -527,19 +527,19 @@ "integrity": "sha512-nqn/0Eg04DxwnkRGSM1xnmGgtfHYOBcEPbFeTu8c1qAbjFEozd6tpw4y6dQrCCL/JLNIRQPsxsUsVnKeWDF/4Q==" }, "@angular/platform-browser": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-9.1.2.tgz", - "integrity": "sha512-/3L4DdvnebvaJUurusaq8RJBFfr/SHWG6DMmV1VVpADxe8kjREyN0LdNDSkZgVf/QcUSwNEA6153iwcF92Otew==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-9.1.3.tgz", + "integrity": "sha512-ts6b7QBp4+UpuQS0Eb+FOT4MZVPUwWtWH4iIfBILJaG/0Bk4jHZeXuLFcpgeZkwMNCZEt9f0JrhnHfvuLCez5g==" }, "@angular/platform-browser-dynamic": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-9.1.2.tgz", - "integrity": "sha512-BWHxy8S71z+NmUQmtR+/Dkohlj3LQIXltOqeVdCSpjV9cultBNN3bE1w0Rjp3BmCRGCIDH7qFlr4U5woHa7ldw==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-9.1.3.tgz", + "integrity": "sha512-9U8PkBflnQtKIZ4t0AqYUaKLg7Kx1BvJ1D/S8FZo/V5kBJK6NkGVu58XddWVc+dtO3V+vCC/VLG7fb2PMf/V5g==" }, "@angular/router": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@angular/router/-/router-9.1.2.tgz", - "integrity": "sha512-csxE4HkuhVR1X932Q3kSDqBoF7Awuq5dsjv0nFk78raiHgG3CNnfMLHt8xD9XtOmR7ZT+D4yh/YmIK6W7J5hbQ==" + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@angular/router/-/router-9.1.3.tgz", + "integrity": "sha512-NmOazeJ2WVT5eTyNHXYxBRAHic6i6Ku5DgsGWBgjUIR/Qmwb/i/xpagbA9WQQZ2OrGbp4ICEYgGkikeIrlQLPA==" }, "@auth0/angular-jwt": { "version": "4.0.0", @@ -1620,12 +1620,12 @@ } }, "@material-ui/react-transition-group": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@material-ui/react-transition-group/-/react-transition-group-4.2.0.tgz", - "integrity": "sha512-4zapZ0gW1ZTws5aH9OGy3IMvtTV/olc7YrVSkM1WFu1FsrEhL+qarEniRjx7LjHt0gukFqoINfElI8v2boVMQA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@material-ui/react-transition-group/-/react-transition-group-4.3.0.tgz", + "integrity": "sha512-CwQ0aXrlUynUTY6sh3UvKuvye1o92en20VGAs6TORnSxUYeRmkX8YeTUN3lAkGiBX1z222FxLFO36WWh6q73rQ==", "requires": { - "@babel/runtime": "^7.4.5", - "dom-helpers": "^3.4.0", + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", "loose-envify": "^1.4.0", "prop-types": "^15.6.2" } @@ -1694,12 +1694,12 @@ "integrity": "sha512-KDvuNBxfTgDHjESiouEoPXq8GCUYbHnA+jOs8MzzwVDmxNLeZuDy3amaVzKv9Q5nncI6mNGcXOzTKWzapET8Sw==" }, "@ngtools/webpack": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-9.1.1.tgz", - "integrity": "sha512-4RPlk6aIlYhk9isTvXbMaA2G0LhxOzcZ+2iG7zV9Yj/Vm8+lrRexpQ/kC/Dh0GI/oCtKIkVpUzx5LTozYeTLdQ==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-9.1.3.tgz", + "integrity": "sha512-yQ7LHqICC1Xf4E4Jw1jDCAJHYwPX7T7mRFxMd+R/0EBDcz1XBSh85+vnVs2PoJHVH+8ONOEkZKKSjABe1CZUvg==", "dev": true, "requires": { - "@angular-devkit/core": "9.1.1", + "@angular-devkit/core": "9.1.3", "enhanced-resolve": "4.1.1", "rxjs": "6.5.4", "webpack-sources": "1.4.3" @@ -1738,23 +1738,23 @@ } }, "@schematics/angular": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@schematics/angular/-/angular-9.1.1.tgz", - "integrity": "sha512-V0DcDNgHQ2YR+PGZI6+pf/mUNNxt5SusShkZ1PbwIMk/HUQpzEGkLjm3v1Jw9eIZKiuDx615GNU1xDzQ/KyNRQ==", + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@schematics/angular/-/angular-9.1.3.tgz", + "integrity": "sha512-3vxrPyvcIWvyWgbx/fpeQvMThXlmcnHTXVn2dpTcE2BzstAYdG17W3qouhvmlqd6m28orxkKKiZaJYMNTtnV/A==", "dev": true, "requires": { - "@angular-devkit/core": "9.1.1", - "@angular-devkit/schematics": "9.1.1" + "@angular-devkit/core": "9.1.3", + "@angular-devkit/schematics": "9.1.3" } }, "@schematics/update": { - "version": "0.901.1", - "resolved": "https://registry.npmjs.org/@schematics/update/-/update-0.901.1.tgz", - "integrity": "sha512-g5B8hjXKCbUNLKCkWQxc5r2D9lOQXRTLZJNIqva+l/ck0AML5MpelxkqQId9ZGLfQqfFk/XjdSHRWgcmUs1WnA==", + "version": "0.901.3", + "resolved": "https://registry.npmjs.org/@schematics/update/-/update-0.901.3.tgz", + "integrity": "sha512-kwfc9LMMi6ryN2oiat9vTqGMK1jKwI1c45Nm1xala35nsbQSQ12qeL3MrVmnSCeI14+cinFUbp8mWI9/3M+/Lg==", "dev": true, "requires": { - "@angular-devkit/core": "9.1.1", - "@angular-devkit/schematics": "9.1.1", + "@angular-devkit/core": "9.1.3", + "@angular-devkit/schematics": "9.1.3", "@yarnpkg/lockfile": "1.1.0", "ini": "1.3.5", "npm-package-arg": "^8.0.0", @@ -1922,9 +1922,9 @@ "integrity": "sha512-13gmo3M2qVvjQrWNseqM3+cR6S2Ss3grbR2NZltgMq94wOwqJYQdgn8qzwDshzgXqMlSUtyPZjysImmktu22ew==" }, "@types/node": { - "version": "13.13.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.1.tgz", - "integrity": "sha512-uysqysLJ+As9jqI5yqjwP3QJrhOcUwBjHUlUxPxjbplwKoILvXVsmYWEhfmAQlrPfbRZmhJB007o4L9sKqtHqQ==", + "version": "13.13.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.2.tgz", + "integrity": "sha512-LB2R1Oyhpg8gu4SON/mfforE525+Hi/M1ineICEDftqNVTyFg1aRIeGuTvXAoWHc4nbrFncWtJgMmoyRvuGh7A==", "dev": true }, "@types/prop-types": { @@ -2695,9 +2695,9 @@ } }, "babel-plugin-dynamic-import-node": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.2.tgz", - "integrity": "sha512-yvczAMjbc73xira9yTyF1XnEmkX8QwlUhmxuhimeMUeAaA6s7busTPRVDzhVG7eeBdNcRiZ/mAwFrJ9It4vQcg==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", "dev": true, "requires": { "object.assign": "^4.1.0" @@ -3223,9 +3223,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001045", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001045.tgz", - "integrity": "sha512-Y8o2Iz1KPcD6FjySbk1sPpvJqchgxk/iow0DABpGyzA1UeQAuxh63Xh0Enj5/BrsYbXtCN32JmR4ZxQTCQ6E6A==", + "version": "1.0.30001046", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001046.tgz", + "integrity": "sha512-CsGjBRYWG6FvgbyGy+hBbaezpwiqIOLkxQPY4A4Ea49g1eNsnQuESB+n4QM0BKii1j80MyJ26Ir5ywTQkbRE4g==", "dev": true }, "canonical-path": { @@ -4169,6 +4169,55 @@ "timsort": "^0.3.0" } }, + "css-loader": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-3.5.1.tgz", + "integrity": "sha512-0G4CbcZzQ9D1Q6ndOfjFuMDo8uLYMu5vc9Abs5ztyHcKvmil6GJrMiNjzzi3tQvUF+mVRuDg7bE6Oc0Prolgig==", + "dev": true, + "requires": { + "camelcase": "^5.3.1", + "cssesc": "^3.0.0", + "icss-utils": "^4.1.1", + "loader-utils": "^1.2.3", + "normalize-path": "^3.0.0", + "postcss": "^7.0.27", + "postcss-modules-extract-imports": "^2.0.0", + "postcss-modules-local-by-default": "^3.0.2", + "postcss-modules-scope": "^2.2.0", + "postcss-modules-values": "^3.0.0", + "postcss-value-parser": "^4.0.3", + "schema-utils": "^2.6.5", + "semver": "^6.3.0" + }, + "dependencies": { + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + } + }, + "loader-utils": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", + "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "css-parse": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-2.0.0.tgz", @@ -4715,11 +4764,12 @@ "integrity": "sha512-hN42DmUgtweBx0iBjDLO4WtKOMcK8yBmPx/fgdsgQadLuzPu/8co3oLdK5yMmeM/vnUd3yDyV6qV8/NzxBexQg==" }, "dom-helpers": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-3.4.0.tgz", - "integrity": "sha512-LnuPJ+dwqKDIyotW1VzmOZ5TONUN7CwkCR5hrgawTUbkBGYdeoNLZo6nNfGkCrjtE1nXXaj7iMMpDa8/d9WoIA==", + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.1.4.tgz", + "integrity": "sha512-TjMyeVUvNEnOnhzs6uAn9Ya47GmMo3qq7m+Lr/3ON0Rs5kHvb8I+SQYjLUSYn7qhEm0QjW0yrBkvz9yOrwwz1A==", "requires": { - "@babel/runtime": "^7.1.2" + "@babel/runtime": "^7.8.7", + "csstype": "^2.6.7" } }, "dom-serialize": { @@ -4823,9 +4873,9 @@ "dev": true }, "electron-to-chromium": { - "version": "1.3.414", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.414.tgz", - "integrity": "sha512-UfxhIvED++qLwWrAq9uYVcqF8FdeV9sU2S7qhiHYFODxzXRrd1GZRl/PjITHsTEejgibcWDraD8TQqoHb1aCBQ==", + "version": "1.3.416", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.416.tgz", + "integrity": "sha512-fmSrpOQC1dEXzsznzAMXbhQLkpAr21WtaUfRXnIbh8kblZIaMwSL6A8u2RZHAzZliSoSOM3FzS2z/j8tVqrAAw==", "dev": true }, "elliptic": { @@ -5565,9 +5615,9 @@ } }, "make-dir": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", - "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", "dev": true, "requires": { "semver": "^6.0.0" @@ -5770,9 +5820,9 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", - "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", "dev": true, "optional": true }, @@ -6296,6 +6346,15 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "icss-utils": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-4.1.1.tgz", + "integrity": "sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==", + "dev": true, + "requires": { + "postcss": "^7.0.14" + } + }, "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", @@ -7077,9 +7136,9 @@ "dev": true }, "jasmine-spec-reporter": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/jasmine-spec-reporter/-/jasmine-spec-reporter-5.0.1.tgz", - "integrity": "sha512-RrOZ+bSPnbk1/9KKs5lm0Nl0cqDCh/XXVlCmu3nkhEJH6HTDh4hoJZu3q8e9aq37C0eXEf/JEJnYy+t4m3arZQ==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/jasmine-spec-reporter/-/jasmine-spec-reporter-5.0.2.tgz", + "integrity": "sha512-6gP1LbVgJ+d7PKksQBc2H0oDGNRQI3gKUsWlswKaQ2fif9X5gzhQcgM5+kiJGCQVurOG09jqNhk7payggyp5+g==", "dev": true, "requires": { "colors": "1.4.0" @@ -8275,18 +8334,18 @@ "dev": true }, "mime-db": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", - "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==", + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==", "dev": true }, "mime-types": { - "version": "2.1.26", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", - "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", "dev": true, "requires": { - "mime-db": "1.43.0" + "mime-db": "1.44.0" } }, "mimic-fn": { @@ -10090,6 +10149,47 @@ } } }, + "postcss-modules-extract-imports": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-2.0.0.tgz", + "integrity": "sha512-LaYLDNS4SG8Q5WAWqIJgdHPJrDDr/Lv775rMBFUbgjTz6j34lUznACHcdRWroPvXANP2Vj7yNK57vp9eFqzLWQ==", + "dev": true, + "requires": { + "postcss": "^7.0.5" + } + }, + "postcss-modules-local-by-default": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.2.tgz", + "integrity": "sha512-jM/V8eqM4oJ/22j0gx4jrp63GSvDH6v86OqyTHHUvk4/k1vceipZsaymiZ5PvocqZOl5SFHiFJqjs3la0wnfIQ==", + "dev": true, + "requires": { + "icss-utils": "^4.1.1", + "postcss": "^7.0.16", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.0.0" + } + }, + "postcss-modules-scope": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz", + "integrity": "sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ==", + "dev": true, + "requires": { + "postcss": "^7.0.6", + "postcss-selector-parser": "^6.0.0" + } + }, + "postcss-modules-values": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz", + "integrity": "sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg==", + "dev": true, + "requires": { + "icss-utils": "^4.0.0", + "postcss": "^7.0.6" + } + }, "postcss-normalize-charset": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-4.0.1.tgz", @@ -10957,17 +11057,6 @@ "dom-helpers": "^5.0.1", "loose-envify": "^1.4.0", "prop-types": "^15.6.2" - }, - "dependencies": { - "dom-helpers": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.1.4.tgz", - "integrity": "sha512-TjMyeVUvNEnOnhzs6uAn9Ya47GmMo3qq7m+Lr/3ON0Rs5kHvb8I+SQYjLUSYn7qhEm0QjW0yrBkvz9yOrwwz1A==", - "requires": { - "@babel/runtime": "^7.8.7", - "csstype": "^2.6.7" - } - } } }, "reactcss": { @@ -11217,9 +11306,9 @@ "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" }, "resolve": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.16.1.tgz", - "integrity": "sha512-rmAglCSqWWMrrBv/XM6sW0NuRFiKViw/W4d9EbC4pt+49H8JwHy+mcGmALTEg504AUDcLTvb1T2q3E9AnmY+ig==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", + "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", "dev": true, "requires": { "path-parse": "^1.0.6" @@ -12187,9 +12276,9 @@ } }, "source-map-support": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz", - "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==", + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -13074,24 +13163,6 @@ "make-error": "^1.1.1", "source-map-support": "^0.5.17", "yn": "3.1.1" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - }, - "source-map-support": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.17.tgz", - "integrity": "sha512-bwdKOBZ5L0gFRh4KOxNap/J/MpvX9Yxsq9lFDx65s3o7F/NiHy7JRaGIS8MwW6tZPAq9UXE207Il0cfcb5yu/Q==", - "dev": true, - "requires": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - } } }, "tslib": { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts index 5794144bec..69faafcb14 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts @@ -42,7 +42,7 @@ import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { AttributeService } from '@core/http/attribute.service'; import { Type } from '@angular/core'; import { TranslateService } from '@ngx-translate/core'; -import { UtilsService } from '@app/core/public-api'; +import { UtilsService } from '@core/services/utils.service'; // @dynamic export class MapWidgetController implements MapWidgetInterface { From 99ef1cba28aa7ba8e04314c25246384f92d17b40 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 24 Apr 2020 18:36:46 +0300 Subject: [PATCH 092/102] IE11 fixes --- ui-ngx/package-lock.json | 6 +++--- ui-ngx/package.json | 2 +- ui-ngx/src/app/core/utils.ts | 4 +++- ui-ngx/src/browserslist | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ui-ngx/package-lock.json b/ui-ngx/package-lock.json index 611439194e..79b698fd47 100644 --- a/ui-ngx/package-lock.json +++ b/ui-ngx/package-lock.json @@ -7730,9 +7730,9 @@ "integrity": "sha1-RGf0n5jRv9VpWb2cZwUgPdJgEnc=" }, "leaflet.gridlayer.googlemutant": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/leaflet.gridlayer.googlemutant/-/leaflet.gridlayer.googlemutant-0.9.0.tgz", - "integrity": "sha512-hkLEHLcdcCaq1OKgUla5ar6fCaUK6yBYIY9HyAclPcMwhnpJTnLmYYPbVJbq3Niz0YiO75gls4A6D6jkBJXpuw==" + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/leaflet.gridlayer.googlemutant/-/leaflet.gridlayer.googlemutant-0.8.0.tgz", + "integrity": "sha512-Ain+jgDKRhlM6qNDDj2QFJa9vXUqV096N0PmpHO3DoNLS4I7EynTQCJXN+9qY4C51ZpV4Q4CI+apNv5XiP5aUA==" }, "leaflet.markercluster": { "version": "1.4.1", diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 8a25206c32..3f8895a9ad 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -61,7 +61,7 @@ "leaflet-geometryutil": "^0.9.3", "leaflet-polylinedecorator": "^1.6.0", "leaflet-providers": "^1.9.1", - "leaflet.gridlayer.googlemutant": "^0.9.0", + "leaflet.gridlayer.googlemutant": "^0.8.0", "leaflet.markercluster": "^1.4.1", "material-design-icons": "^3.0.1", "messageformat": "^2.3.0", diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index b0ab617a42..ea3682f72e 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -539,7 +539,9 @@ export function parseTemplate(template: string, data: object, translateFn?: (key } const expressions = template.match(/\{(.*?)\}/g); if (expressions) { - const clearMatches = template.match(/(?<=\{)(.+?)(?=(\}|\:))/g); + // TODO: not supported in IE + // const clearMatches = template.match(/(?<=\{)(.+?)(?=(\}|\:))/g); + const clearMatches = template.match(/\{(.+?)(\}|\:)/g); for (const key in data) { if (!key.includes('|')) variables += `let ${key} = '${clearMatches[key] ? padValue(data[key], +clearMatches[key]) : data[key]}';`; diff --git a/ui-ngx/src/browserslist b/ui-ngx/src/browserslist index 37371cb04b..b744b0a114 100644 --- a/ui-ngx/src/browserslist +++ b/ui-ngx/src/browserslist @@ -8,4 +8,4 @@ last 2 versions Firefox ESR not dead -not IE 9-11 \ No newline at end of file +IE 10-11 From a779839081b92a995afa67427b53bdfefe34dc92 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 19:16:25 +0300 Subject: [PATCH 093/102] Fixed transport --- .../org/thingsboard/server/actors/ActorSystemContext.java | 3 +++ .../common/transport/service/DefaultTransportService.java | 6 ++++++ docker/tb-coap-transport.env | 2 ++ docker/tb-http-transport.env | 2 ++ docker/tb-mqtt-transport.env | 2 ++ 5 files changed, 15 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e9b68882b..28f5b188e5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,6 +32,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -233,6 +234,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-core mode */ + @Lazy @Autowired(required = false) @Getter private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; @@ -240,6 +242,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-rule-engine mode */ + @Lazy @Autowired(required = false) @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index c820167de1..3db7300bc3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -544,6 +544,9 @@ public class DefaultTransportService implements TransportService { protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg); + } tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? @@ -552,6 +555,9 @@ public class DefaultTransportService implements TransportService { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg); + } ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index af95bc904d..8a75904185 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 71bc434241..e62424290f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 163cfab451..54c35f355c 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 From 38ebbe257656047da3f627cda091b466166f8427 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Fri, 24 Apr 2020 19:42:32 +0300 Subject: [PATCH 094/102] Revert "Merge with master. AlarmRepository.findAlarms is failing (#2663)" This reverts commit 2c02406ee5f41b08e655b634dd9871403f67149d. --- .../data/json/demo/dashboards/gateways.json | 1274 ----------------- .../dashboards/rule_engine_statistics.json | 520 ------- .../data/upgrade/2.4.3/schema_update_ttl.sql | 25 - .../server/actors/ActorSystemContext.java | 122 +- .../server/actors/app/AppActor.java | 104 +- .../server/actors/device/DeviceActor.java | 13 +- .../device/DeviceActorMessageProcessor.java | 183 ++- .../device/DeviceActorToRuleEngineMsg.java | 21 +- .../actors/rpc/BasicRpcSessionListener.java | 83 ++ .../server/actors/rpc/RpcBroadcastMsg.java | 13 +- .../server/actors/rpc/RpcManagerActor.java | 230 +++ .../server/actors/rpc/RpcSessionActor.java | 135 ++ .../actors/rpc/RpcSessionClosedMsg.java | 29 + .../actors/rpc/RpcSessionConnectedMsg.java | 31 + .../rpc/RpcSessionCreateRequestMsg.java} | 18 +- .../actors/rpc/RpcSessionDisconnectedMsg.java | 29 + .../server/actors/rpc/RpcSessionTellMsg.java | 15 +- .../server/actors/rpc/SessionActorInfo.java | 30 + .../actors/ruleChain/DefaultTbContext.java | 234 ++- .../RemoteToRuleChainTellNextMsg.java | 48 + .../actors/ruleChain/RuleChainActor.java | 31 +- .../RuleChainActorMessageProcessor.java | 237 ++- .../ruleChain/RuleChainManagerActor.java | 68 +- .../ruleChain/RuleChainToRuleChainMsg.java | 1 + .../actors/ruleChain/RuleNodeActor.java | 4 - .../RuleNodeActorMessageProcessor.java | 8 +- .../RuleNodeToRuleChainTellNextMsg.java | 1 - .../server/actors/service/ActorService.java | 19 +- .../server/actors/service/ComponentActor.java | 6 +- .../actors/service/ContextAwareActor.java | 4 +- .../actors/service/DefaultActorService.java | 204 ++- .../actors/shared/ComponentMsgProcessor.java | 11 +- .../actors/shared/EntityActorsManager.java | 87 ++ .../shared/rulechain/RuleChainManager.java | 59 + .../rulechain/SystemRuleChainManager.java | 48 + .../rulechain/TenantRuleChainManager.java | 53 + .../server/actors/stats/StatsActor.java | 7 +- .../server/actors/tenant/TenantActor.java | 161 +-- .../server/config/WebSocketConfiguration.java | 5 +- .../server/controller/AdminController.java | 14 +- .../server/controller/AlarmController.java | 4 +- .../server/controller/AssetController.java | 4 +- .../server/controller/AuditLogController.java | 2 - .../server/controller/AuthController.java | 2 - .../server/controller/BaseController.java | 45 +- .../ComponentDescriptorController.java | 2 - .../server/controller/CustomerController.java | 2 - .../controller/DashboardController.java | 3 +- .../server/controller/DeviceController.java | 16 +- .../controller/EntityRelationController.java | 2 - .../controller/EntityViewController.java | 2 - .../server/controller/EventController.java | 2 - .../server/controller/QueueController.java | 54 - .../server/controller/RpcController.java | 14 +- .../controller/RuleChainController.java | 17 +- .../controller/TelemetryController.java | 20 +- .../server/controller/TenantController.java | 6 +- .../server/controller/UserController.java | 2 - .../controller/WidgetTypeController.java | 2 - .../controller/WidgetsBundleController.java | 3 +- .../controller/plugin/TbWebSocketHandler.java | 9 +- .../install/ThingsboardInstallService.java | 7 +- .../CurrentServerInstanceService.java | 55 + .../cluster}/discovery/DiscoveryService.java | 15 +- .../discovery/DiscoveryServiceListener.java | 28 + .../discovery/DummyDiscoveryService.java | 42 +- .../cluster/discovery/ServerInstance.java | 47 + .../discovery/ServerInstanceService.java} | 10 +- .../discovery/ZkDiscoveryService.java | 227 +-- .../routing/ClusterRoutingService.java | 23 +- .../ConsistentClusterRoutingService.java | 153 ++ .../routing}/ConsistentHashCircle.java | 16 +- .../cluster/rpc/ClusterGrpcService.java | 161 +++ .../cluster/rpc/ClusterRpcService.java | 42 + .../service/cluster/rpc/GrpcSession.java | 125 ++ .../rpc/GrpcSessionListener.java} | 20 +- .../service/cluster/rpc/RpcMsgListener.java | 32 + .../encoding/DataDecodingEncodingService.java | 5 + .../service/encoding/ProtoWithFSTService.java | 24 +- .../environment/EnvironmentLogService.java | 4 +- .../ClusterRpcCallbackExecutorService.java | 17 +- .../install/SqlDatabaseUpgradeService.java | 9 - .../CassandraEntitiesToSqlMigrateService.java | 7 +- .../migrate/CassandraToSqlEventTsColumn.java | 40 - .../queue/DefaultTbClusterService.java | 204 --- .../queue/DefaultTbCoreConsumerService.java | 290 ---- .../DefaultTbRuleEngineConsumerService.java | 262 ---- .../DefaultTenantRoutingInfoService.java | 47 - .../service/queue/TbClusterService.java | 52 - .../service/queue/TbCoreConsumerService.java | 23 - .../service/queue/TbMsgPackCallback.java | 48 - .../queue/TbMsgPackProcessingContext.java | 84 -- .../queue/TbPackProcessingContext.java | 90 -- .../queue/TbRuleEngineConsumerService.java | 23 - .../queue/TbRuleEngineConsumerStats.java | 132 -- .../queue/TbTenantRuleEngineStats.java | 92 -- .../processing/AbstractConsumerService.java | 149 -- .../AbstractTbRuleEngineSubmitStrategy.java | 71 - .../BatchTbRuleEngineSubmitStrategy.java | 86 -- .../BurstTbRuleEngineSubmitStrategy.java | 50 - ...lByEntityIdTbRuleEngineSubmitStrategy.java | 108 -- ...riginatorIdTbRuleEngineSubmitStrategy.java | 44 - .../SequentialTbRuleEngineSubmitStrategy.java | 73 - .../TbRuleEngineProcessingResult.java | 58 - .../TbRuleEngineProcessingStrategy.java | 22 - ...TbRuleEngineProcessingStrategyFactory.java | 140 -- .../TbRuleEngineSubmitStrategy.java | 39 - .../TbRuleEngineSubmitStrategyFactory.java | 43 - .../service/rpc/DefaultDeviceRpcService.java | 222 +++ .../rpc/DefaultTbCoreDeviceRpcService.java | 199 --- .../rpc/DefaultTbRuleEngineRpcService.java | 177 --- .../server/service/rpc/DeviceRpcService.java | 41 + .../service/rpc/FromDeviceRpcResponse.java | 1 + .../service/rpc/TbCoreDeviceRpcService.java | 57 - .../rpc/ToDeviceRpcRequestActorMsg.java | 5 +- .../rpc/ToServerRpcResponseActorMsg.java | 41 +- .../script/AbstractJsInvokeService.java | 48 +- .../AbstractNashornJsInvokeService.java | 4 +- .../script/NashornJsInvokeService.java | 10 - .../service/script/RemoteJsInvokeService.java | 99 +- .../script/RemoteJsRequestEncoder.java | 9 +- .../script/RemoteJsResponseDecoder.java | 12 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../auth/rest/RestAuthenticationProvider.java | 1 - .../device/DefaultDeviceAuthService.java | 14 +- .../permission/AccessControlService.java | 2 + .../DefaultAccessControlService.java | 7 + .../DefaultDeviceSessionCacheService.java | 3 +- .../state/DefaultDeviceStateService.java | 439 +++--- .../service/state/DeviceStateService.java | 12 +- .../DefaultRuleEngineStatisticsService.java | 141 -- .../stats/RuleEngineStatisticsService.java | 23 - .../DefaultSubscriptionManagerService.java | 380 ----- .../DefaultTbLocalSubscriptionService.java | 228 --- .../SubscriptionManagerService.java | 38 - .../subscription/TbAttributeSubscription.java | 50 - .../TbLocalSubscriptionService.java | 36 - .../service/subscription/TbSubscription.java | 52 - .../subscription/TbSubscriptionUtils.java | 247 ---- .../TbTimeseriesSubscription.java | 51 - .../DefaultTelemetrySubscriptionService.java | 675 ++++++++- .../DefaultTelemetryWebSocketService.java | 74 +- .../TelemetrySubscriptionService.java | 27 +- .../service/telemetry/sub/Subscription.java | 80 ++ .../telemetry/sub/SubscriptionUpdate.java | 2 +- .../BaseRuleChainTransactionService.java | 242 ++++ .../transaction/TbTransactionTask.java | 44 + .../DefaultTbCoreToTransportService.java | 87 -- ...ice.java => LocalTransportApiService.java} | 33 +- .../transport/LocalTransportService.java | 227 +++ .../RemoteRuleEngineTransportService.java | 251 ++++ .../transport/RemoteTransportApiService.java | 111 ++ .../RuleEngineStats.java} | 47 +- .../transport/RuleEngineTransportService.java | 31 + .../transport/TbCoreTransportApiService.java | 100 -- .../ToRuleEngineMsgDecoder.java} | 20 +- ...ervice.java => ToTransportMsgEncoder.java} | 17 +- .../transport/TransportApiRequestDecoder.java | 31 + .../TransportApiResponseEncoder.java | 30 + .../transport/TransportApiService.java | 8 +- .../msg/TransportToDeviceActorMsgWrapper.java | 5 +- ... => AbstractTimeseriesCleanUpService.java} | 40 +- .../PsqlTimeseriesCleanUpService.java | 2 +- .../TimescaleTimeseriesCleanUpService.java | 2 +- .../ttl/events/EventsCleanUpService.java | 59 - .../AbstractTimeseriesCleanUpService.java | 49 - .../service/update/DefaultUpdateService.java | 2 - .../thingsboard/server/utils}/JsonUtils.java | 2 +- application/src/main/proto/cluster.proto | 146 ++ .../src/main/proto/jsinvoke.proto | 0 application/src/main/resources/logback.xml | 3 - .../src/main/resources/thingsboard.yml | 286 ++-- .../controller/AbstractControllerTest.java | 6 +- .../controller/BaseAssetControllerTest.java | 74 +- .../BaseEntityViewControllerTest.java | 22 +- .../controller/ControllerSqlTestSuite.java | 7 - .../server/mqtt/MqttNoSqlTestSuite.java | 7 - .../server/mqtt/MqttSqlTestSuite.java | 7 - ...tractMqttServerSideRpcIntegrationTest.java | 19 +- .../AbstractMqttTelemetryIntegrationTest.java | 24 +- .../server/rules/RuleEngineSqlTestSuite.java | 7 - ...AbstractRuleEngineFlowIntegrationTest.java | 50 +- ...actRuleEngineLifecycleIntegrationTest.java | 27 +- .../ConsistentClusterRoutingServiceTest.java | 105 ++ .../ConsistentHashParitionServiceTest.java | 118 -- .../queue/TbMsgPackProcessingContextTest.java | 66 - .../script/RuleNodeJsScriptEngineTest.java | 15 +- .../script/TestNashornJsInvokeService.java | 5 - .../server/system/SystemNoSqlTestSuite.java | 48 - .../server/system/SystemSqlTestSuite.java | 7 - .../server/dao/alarm/AlarmService.java | 2 +- .../thingsboard/server/dao/util/HsqlDao.java | 4 - .../server/dao/util/NoSqlAnyDao.java | 4 - .../server/dao/util/NoSqlTsDao.java | 4 - .../thingsboard/server/dao/util/PsqlDao.java | 4 - .../server/dao/util/PsqlTsAnyDao.java | 4 - .../thingsboard/server/dao/util/SqlDao.java | 4 +- .../server/dao/util/SqlTsAnyDao.java | 4 - .../thingsboard/server/dao/util/SqlTsDao.java | 4 - .../server/dao/util/TimescaleDBTsDao.java | 4 - .../server/common/data/Tenant.java | 22 - .../server/common/data/alarm/Alarm.java | 1 - .../common/data/{id => alarm}/AlarmId.java | 4 +- .../common/data/id/EntityIdFactory.java | 1 + .../common/data/kv/BaseAttributeKvEntry.java | 4 - .../server/common/msg/MsgType.java | 21 +- .../thingsboard/server/common/msg/TbMsg.java | 103 +- .../common/msg/TbMsgTransactionData.java | 30 + .../ClusterEventMsg.java} | 15 +- .../common/msg/cluster/SendToClusterMsg.java | 25 +- .../common/msg/cluster/ServerAddress.java | 39 +- .../server/common/msg/cluster/ServerType.java | 9 +- .../msg/core/ToServerRpcResponseMsg.java} | 16 +- .../msg/plugin/ComponentLifecycleMsg.java | 2 +- .../common/msg/queue/RuleEngineException.java | 38 - .../common/msg/queue/RuleNodeException.java | 58 - .../server/common/msg/queue/ServiceQueue.java | 62 - .../server/common/msg/queue/ServiceType.java | 25 - .../common/msg/queue/TbMsgCallback.java | 37 - .../common/msg/queue/TopicPartitionInfo.java | 80 -- .../ServiceToRuleEngineMsg.java} | 13 +- .../DeviceActorClientSideRpcTimeoutMsg.java | 33 + common/message/src/main/proto/tbmsg.proto | 9 +- common/queue/pom.xml | 37 - .../AbstractTbKafkaTemplate.java} | 24 +- .../AsyncCallbackTemplate.java | 2 +- .../server/kafka/TBKafkaAdmin.java | 69 + .../server/kafka/TBKafkaConsumerTemplate.java | 88 ++ .../server/kafka/TBKafkaProducerTemplate.java | 131 ++ .../{queue => }/kafka/TbKafkaDecoder.java | 6 +- .../{queue => }/kafka/TbKafkaEncoder.java | 2 +- .../TbKafkaHandler.java} | 4 +- .../TbKafkaPartitioner.java} | 23 +- .../{queue => }/kafka/TbKafkaProperty.java | 6 +- .../TbKafkaRequestIdExtractor.java} | 9 +- .../TbKafkaRequestTemplate.java} | 141 +- .../server/kafka/TbKafkaResponseTemplate.java | 161 +++ .../{queue => }/kafka/TbKafkaSettings.java | 26 +- .../TbNodeIdProvider.java} | 41 +- .../server/queue/TbQueueCallback.java | 23 - .../server/queue/TbQueueMsgDecoder.java | 24 - .../server/queue/TbQueueMsgHeaders.java | 27 - .../server/queue/TbQueueMsgMetadata.java | 19 - .../server/queue/TbQueueProducer.java | 29 - .../server/queue/TbQueueRequestTemplate.java | 28 - .../server/queue/TbQueueResponseTemplate.java | 23 - .../azure/servicebus/TbServiceBusAdmin.java | 77 - .../TbServiceBusConsumerTemplate.java | 210 --- .../TbServiceBusProducerTemplate.java | 110 -- .../servicebus/TbServiceBusSettings.java | 37 - .../queue/common/DefaultTbQueueMsg.java | 37 - .../common/DefaultTbQueueMsgHeaders.java | 41 - .../DefaultTbQueueResponseTemplate.java | 163 --- .../MultipleTbQueueTbMsgCallbackWrapper.java | 47 - .../queue/common/TbProtoJsQueueMsg.java | 43 - .../server/queue/common/TbProtoQueueMsg.java | 55 - .../common/TbQueueTbMsgCallbackWrapper.java | 41 - .../discovery/ClusterTopologyChangeEvent.java | 34 - .../ConsistentHashPartitionService.java | 339 ----- .../DefaultTbServiceInfoProvider.java | 119 -- .../queue/discovery/PartitionChangeEvent.java | 43 - .../queue/discovery/PartitionService.java | 58 - .../queue/discovery/TenantRoutingInfo.java | 26 - .../discovery/TenantRoutingInfoService.java | 23 - .../discovery/TopicPartitionInfoKey.java | 44 - .../server/queue/kafka/KafkaTbQueueMsg.java | 54 - .../queue/kafka/KafkaTbQueueMsgMetadata.java | 27 - .../server/queue/kafka/TbKafkaAdmin.java | 89 -- .../queue/kafka/TbKafkaConsumerTemplate.java | 159 -- .../queue/kafka/TbKafkaProducerTemplate.java | 112 -- .../queue/kafka/TbKafkaTopicConfigs.java | 71 - .../server/queue/memory/InMemoryStorage.java | 80 -- .../queue/memory/InMemoryTbQueueConsumer.java | 98 -- .../queue/memory/InMemoryTbQueueProducer.java | 58 - .../provider/AwsSqsMonolithQueueFactory.java | 169 --- .../provider/AwsSqsTbCoreQueueFactory.java | 158 -- .../AwsSqsTbRuleEngineQueueFactory.java | 135 -- .../provider/AwsSqsTransportQueueFactory.java | 124 -- .../InMemoryMonolithQueueFactory.java | 123 -- .../InMemoryTbTransportQueueFactory.java | 99 -- .../provider/KafkaMonolithQueueFactory.java | 250 ---- .../provider/KafkaTbCoreQueueFactory.java | 221 --- .../KafkaTbRuleEngineQueueFactory.java | 193 --- .../KafkaTbTransportQueueFactory.java | 138 -- .../provider/PubSubMonolithQueueFactory.java | 177 --- .../provider/PubSubTbCoreQueueFactory.java | 149 -- .../PubSubTbRuleEngineQueueFactory.java | 138 -- .../provider/PubSubTransportQueueFactory.java | 132 -- .../RabbitMqMonolithQueueFactory.java | 169 --- .../provider/RabbitMqTbCoreQueueFactory.java | 158 -- .../RabbitMqTbRuleEngineQueueFactory.java | 136 -- .../RabbitMqTransportQueueFactory.java | 129 -- .../ServiceBusMonolithQueueFactory.java | 142 -- .../ServiceBusTbCoreQueueFactory.java | 124 -- .../ServiceBusTbRuleEngineQueueFactory.java | 107 -- .../ServiceBusTransportQueueFactory.java | 98 -- .../queue/provider/TbCoreQueueFactory.java | 102 -- .../provider/TbCoreQueueProducerProvider.java | 74 - .../provider/TbQueueProducerProvider.java | 66 - .../TbRuleEngineProducerProvider.java | 75 - .../provider/TbRuleEngineQueueFactory.java | 89 -- .../provider/TbTransportQueueFactory.java | 38 - .../TbTransportQueueProducerProvider.java | 68 - .../server/queue/pubsub/TbPubSubAdmin.java | 186 --- .../pubsub/TbPubSubConsumerTemplate.java | 219 --- .../pubsub/TbPubSubProducerTemplate.java | 134 -- .../server/queue/pubsub/TbPubSubSettings.java | 58 - .../pubsub/TbPubSubSubscriptionSettings.java | 71 - .../queue/rabbitmq/TbRabbitMqAdmin.java | 80 -- .../rabbitmq/TbRabbitMqConsumerTemplate.java | 173 --- .../rabbitmq/TbRabbitMqProducerTemplate.java | 113 -- .../rabbitmq/TbRabbitMqQueueArguments.java | 98 -- .../queue/rabbitmq/TbRabbitMqSettings.java | 65 - .../queue/settings/TbQueueCoreSettings.java | 31 - .../TbQueueRemoteJsInvokeSettings.java | 42 - .../settings/TbQueueRuleEngineSettings.java | 49 - .../settings/TbQueueTransportApiSettings.java | 47 - .../TbRuleEngineQueueConfiguration.java | 31 - .../queue/sqs/AwsSqsTbQueueMsgMetadata.java | 28 - .../server/queue/sqs/TbAwsSqsAdmin.java | 76 - .../queue/sqs/TbAwsSqsConsumerTemplate.java | 231 --- .../queue/sqs/TbAwsSqsProducerTemplate.java | 119 -- .../queue/sqs/TbAwsSqsQueueAttributes.java | 83 -- .../server/queue/util/TbCoreComponent.java | 26 - .../queue/util/TbRuleEngineComponent.java | 26 - common/queue/src/main/proto/queue.proto | 410 ------ .../transport/coap/CoapTransportContext.java | 2 +- .../transport/coap/CoapTransportResource.java | 2 - .../transport/coap/CoapTransportService.java | 8 +- .../transport/http/DeviceApiController.java | 4 +- .../transport/http/HttpTransportContext.java | 2 +- .../transport/mqtt/MqttTransportContext.java | 12 +- .../transport/mqtt/MqttTransportHandler.java | 24 +- .../transport/mqtt/MqttTransportService.java | 2 +- .../mqtt/session/GatewayDeviceSessionCtx.java | 2 - .../mqtt/session/GatewaySessionHandler.java | 6 +- common/transport/transport-api/pom.xml | 17 +- .../transport/SessionMsgProcessor.java} | 9 +- .../common/transport/TransportContext.java | 13 +- .../common/transport/TransportService.java | 14 +- .../service/AbstractTransportService.java | 327 +++++ .../service/DefaultTransportService.java | 600 -------- .../service/RemoteTransportService.java | 339 +++++ .../service/ToRuleEngineMsgEncoder.java | 2 +- .../ToTransportMsgResponseDecoder.java | 8 +- .../service/TransportApiRequestEncoder.java | 2 +- .../service/TransportApiResponseDecoder.java | 8 +- .../TransportTenantRoutingInfoService.java | 52 - .../session/DeviceAwareSessionContext.java | 8 +- .../src/main/proto/transport.proto | 244 ++++ .../server/dao/alarm/BaseAlarmService.java | 2 +- .../server/dao/entity/BaseEntityService.java | 2 +- .../server/dao/model/BaseEntity.java | 4 +- .../server/dao/model/BaseSqlEntity.java | 5 +- .../server/dao/model/ModelConstants.java | 5 - .../dao/model/sql/AbstractAlarmEntity.java | 4 +- .../dao/model/sql/AbstractAssetEntity.java | 2 +- .../dao/model/sql/AbstractDeviceEntity.java | 6 +- .../model/sql/AbstractEntityViewEntity.java | 7 +- .../dao/model/sql/AdminSettingsEntity.java | 2 +- .../server/dao/model/sql/AlarmEntity.java | 1 - .../server/dao/model/sql/AuditLogEntity.java | 6 +- .../model/sql/ComponentDescriptorEntity.java | 5 +- .../server/dao/model/sql/CustomerEntity.java | 6 +- .../server/dao/model/sql/DashboardEntity.java | 6 +- .../dao/model/sql/DashboardInfoEntity.java | 6 +- .../model/sql/DeviceCredentialsEntity.java | 6 +- .../server/dao/model/sql/EventEntity.java | 20 +- .../server/dao/model/sql/RuleChainEntity.java | 6 +- .../server/dao/model/sql/RuleNodeEntity.java | 6 +- .../server/dao/model/sql/TenantEntity.java | 16 +- .../dao/model/sql/UserCredentialsEntity.java | 6 +- .../server/dao/model/sql/UserEntity.java | 6 +- .../dao/model/sql/WidgetTypeEntity.java | 6 +- .../dao/model/sql/WidgetsBundleEntity.java | 2 +- .../server/dao/sql/JpaAbstractDao.java | 4 +- .../server/dao/sql/alarm/AlarmRepository.java | 18 +- .../server/dao/sql/asset/JpaAssetDao.java | 2 + .../server/dao/sql/audit/JpaAuditLogDao.java | 3 - ...ctComponentDescriptorInsertRepository.java | 8 +- ...qlComponentDescriptorInsertRepository.java | 2 +- .../JpaBaseComponentDescriptorDao.java | 3 + .../dao/sql/customer/JpaCustomerDao.java | 3 + .../sql/dashboard/JpaDashboardInfoDao.java | 3 + .../dao/sql/entityview/JpaEntityViewDao.java | 8 +- .../event/AbstractEventInsertRepository.java | 5 +- .../sql/event/HsqlEventInsertRepository.java | 6 +- .../server/dao/sql/event/JpaBaseEventDao.java | 8 +- .../sql/event/PsqlEventInsertRepository.java | 2 +- .../dao/sql/relation/JpaRelationDao.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 3 + .../server/dao/sql/tenant/JpaTenantDao.java | 5 + .../server/dao/sql/user/JpaUserDao.java | 2 + .../dao/sql/widget/JpaWidgetsBundleDao.java | 2 + ...stractChunkedAggregationTimeseriesDao.java | 4 +- .../timescale/TimescaleTimeseriesDao.java | 4 +- .../server/dao/tenant/TenantServiceImpl.java | 36 +- .../resources/sql/schema-entities-hsql.sql | 6 +- .../main/resources/sql/schema-entities.sql | 30 +- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- .../server/dao/sql/alarm/JpaAlarmDaoTest.java | 2 +- .../test/resources/cassandra-test.properties | 1 - dao/src/test/resources/sql-test.properties | 15 +- .../api/jsInvokeMessageProcessor.js | 5 +- pom.xml | 78 +- .../thingsboard/rest/client/RestClient.java | 23 +- .../api/RuleChainTransactionService.java | 22 +- .../api/RuleEngineDeviceRpcRequest.java | 5 +- .../rule/engine/api/RuleEngineRpcService.java | 6 +- .../api/RuleEngineTelemetryService.java | 2 + .../rule/engine/api/TbContext.java | 92 +- .../thingsboard/rule/engine/api/TbNode.java | 4 +- rule-engine/rule-engine-components/pom.xml | 4 +- .../engine/action/TbAbstractAlarmNode.java | 12 +- .../action/TbAbstractCustomerActionNode.java | 6 +- .../action/TbAbstractRelationActionNode.java | 12 +- .../TbCopyAttributesToEntityViewNode.java | 8 +- .../rule/engine/action/TbLogNode.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 7 +- .../TbSaveToCustomCassandraTableNode.java | 6 +- .../rule/engine/aws/sns/TbSnsNode.java | 7 +- .../rule/engine/aws/sqs/TbSqsNode.java | 9 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 6 +- .../rule/engine/delay/TbMsgDelayNode.java | 7 +- .../engine/filter/TbCheckAlarmStatusNode.java | 3 +- .../engine/filter/TbMsgTypeFilterNode.java | 2 +- .../engine/filter/TbMsgTypeSwitchNode.java | 2 +- .../filter/TbOriginatorTypeFilterNode.java | 2 +- .../rule/engine/flow/TbAckNode.java | 57 - .../rule/engine/flow/TbCheckpointNode.java | 59 - .../rule/engine/gcp/pubsub/TbPubSubNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 30 +- .../rule/engine/kafka/TbKafkaNode.java | 21 +- .../rule/engine/mail/TbSendEmailNode.java | 3 +- .../metadata/TbAbstractGetAttributesNode.java | 3 +- .../TbAbstractGetEntityDetailsNode.java | 2 +- .../engine/metadata/TbEntityGetAttrNode.java | 5 +- .../engine/metadata/TbGetDeviceAttrNode.java | 2 +- .../metadata/TbGetOriginatorFieldsNode.java | 4 +- .../engine/metadata/TbGetTelemetryNode.java | 3 +- .../rule/engine/mqtt/TbMqttNode.java | 4 +- .../rule/engine/rabbitmq/TbRabbitMqNode.java | 4 +- .../rule/engine/rest/TbHttpClient.java | 25 +- .../engine/rest/TbRedisQueueProcessor.java | 125 ++ .../rule/engine/rest/TbRestApiCallNode.java | 19 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 11 +- .../rule/engine/rpc/TbSendRPCRequestNode.java | 24 +- .../rpc/TbSendRpcReplyNodeConfiguration.java | 24 +- .../engine/telemetry/TbMsgAttributesNode.java | 4 + .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +- .../telemetry/TelemetryNodeCallback.java | 4 +- .../TbSynchronizationBeginNode.java | 26 +- .../transaction/TbSynchronizationEndNode.java | 13 +- .../transform/TbAbstractTransformNode.java | 2 +- .../EntitiesAlarmOriginatorIdAsyncLoader.java | 2 +- .../util/EntitiesFieldsAsyncLoader.java | 2 +- .../util/EntitiesTenantIdAsyncLoader.java | 2 +- .../rule/engine/action/TbAlarmNodeTest.java | 50 +- .../engine/filter/TbJsFilterNodeTest.java | 18 +- .../engine/filter/TbJsSwitchNodeTest.java | 12 +- .../engine/mail/TbMsgToEmailNodeTest.java | 4 +- .../TbGetCustomerAttributeNodeTest.java | 34 +- .../transform/TbChangeOriginatorNodeTest.java | 7 +- .../transform/TbTransformMsgNodeTest.java | 19 +- tools/pom.xml | 2 +- .../ThingsboardCoapTransportApplication.java | 2 +- .../src/main/resources/tb-coap-transport.yml | 159 +- .../ThingsboardHttpTransportApplication.java | 2 +- .../src/main/resources/tb-http-transport.yml | 159 +- .../ThingsboardMqttTransportApplication.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 177 +-- ui-ngx/src/app/core/http/public-api.ts | 1 - ui-ngx/src/app/core/http/queue.service.ts | 36 - .../home/pages/tenant/tenant.component.html | 10 - .../home/pages/tenant/tenant.component.scss | 22 - .../home/pages/tenant/tenant.component.ts | 7 +- .../queue/queue-type-list.component.html | 43 - .../queue/queue-type-list.component.ts | 177 --- ui-ngx/src/app/shared/models/public-api.ts | 1 - ui-ngx/src/app/shared/models/queue.models.ts | 22 - ui-ngx/src/app/shared/models/tenant.model.ts | 2 - ui-ngx/src/app/shared/shared.module.ts | 3 - .../assets/locale/locale.constant-en_US.json | 12 +- .../assets/locale/locale.constant-ru_RU.json | 5 - .../assets/locale/locale.constant-uk_UA.json | 5 - ui/package-lock.json | 225 +-- ui/src/app/api/queue.service.js | 40 - ui/src/app/components/queue/index.js | 23 - .../queue/queue-type-list.directive.js | 111 -- .../app/components/queue/queue-type-list.scss | 15 - .../components/queue/queue-type-list.tpl.html | 43 - ui/src/app/layout/index.js | 2 - ui/src/app/locale/locale.constant-en_US.json | 12 +- ui/src/app/locale/locale.constant-ru_RU.json | 6 - ui/src/app/locale/locale.constant-uk_UA.json | 6 - ui/src/app/tenant/tenant-fieldset.tpl.html | 64 +- 497 files changed, 8022 insertions(+), 20215 deletions(-) delete mode 100644 application/src/main/data/json/demo/dashboards/gateways.json delete mode 100644 application/src/main/data/json/demo/dashboards/rule_engine_statistics.json rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java => application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java (57%) create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java rename common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java => application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java (73%) create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java rename application/src/main/java/org/thingsboard/server/{service/queue/processing/TbRuleEngineProcessingDecision.java => actors/rpc/RpcSessionCreateRequestMsg.java} (60%) create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java rename common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java => application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java (73%) create mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/controller/QueueController.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java rename {common/queue/src/main/java/org/thingsboard/server/queue => application/src/main/java/org/thingsboard/server/service/cluster}/discovery/DiscoveryService.java (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java rename {common/queue/src/main/java/org/thingsboard/server/queue => application/src/main/java/org/thingsboard/server/service/cluster}/discovery/DummyDiscoveryService.java (58%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java rename application/src/main/java/org/thingsboard/server/service/{subscription/TbAttributeSubscriptionScope.java => cluster/discovery/ServerInstanceService.java} (79%) rename {common/queue/src/main/java/org/thingsboard/server/queue => application/src/main/java/org/thingsboard/server/service/cluster}/discovery/ZkDiscoveryService.java (67%) rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java => application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java (54%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java rename {common/queue/src/main/java/org/thingsboard/server/queue/discovery => application/src/main/java/org/thingsboard/server/service/cluster/routing}/ConsistentHashCircle.java (69%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java rename application/src/main/java/org/thingsboard/server/service/{rpc/TbRuleEngineDeviceRpcService.java => cluster/rpc/GrpcSessionListener.java} (59%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java rename {common/queue/src/main/java/org/thingsboard/server/queue => application/src/main/java/org/thingsboard/server/service}/environment/EnvironmentLogService.java (90%) rename common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java => application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java (63%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java rename common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java => application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java (50%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java rename application/src/main/java/org/thingsboard/server/service/transport/{DefaultTransportApiService.java => LocalTransportApiService.java} (75%) create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java rename application/src/main/java/org/thingsboard/server/service/{queue/TbCoreConsumerStats.java => transport/RuleEngineStats.java} (63%) create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java rename application/src/main/java/org/thingsboard/server/service/{queue/processing/IdMsgPair.java => transport/ToRuleEngineMsgDecoder.java} (64%) rename application/src/main/java/org/thingsboard/server/service/transport/{TbCoreToTransportService.java => ToTransportMsgEncoder.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java rename application/src/main/java/org/thingsboard/server/service/ttl/{AbstractCleanUpService.java => AbstractTimeseriesCleanUpService.java} (67%) rename application/src/main/java/org/thingsboard/server/service/ttl/{timeseries => }/PsqlTimeseriesCleanUpService.java (96%) rename application/src/main/java/org/thingsboard/server/service/ttl/{timeseries => }/TimescaleTimeseriesCleanUpService.java (95%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java rename {common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util => application/src/main/java/org/thingsboard/server/utils}/JsonUtils.java (97%) create mode 100644 application/src/main/proto/cluster.proto rename {common/queue => application}/src/main/proto/jsinvoke.proto (100%) create mode 100644 application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java delete mode 100644 application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java rename common/data/src/main/java/org/thingsboard/server/common/data/{id => alarm}/AlarmId.java (88%) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java rename common/message/src/main/java/org/thingsboard/server/common/msg/{queue/PartitionChangeMsg.java => cluster/ClusterEventMsg.java} (71%) rename application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java => common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java (55%) rename application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java => common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java (50%) rename application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java => common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java (82%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java => message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java} (76%) delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java rename common/message/src/main/java/org/thingsboard/server/common/msg/{queue/QueueToRuleEngineMsg.java => system/ServiceToRuleEngineMsg.java} (73%) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java rename common/queue/src/main/java/org/thingsboard/server/{queue/common/AbstractTbQueueTemplate.java => kafka/AbstractTbKafkaTemplate.java} (70%) rename common/queue/src/main/java/org/thingsboard/server/{queue/common => kafka}/AsyncCallbackTemplate.java (98%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java rename common/queue/src/main/java/org/thingsboard/server/{queue => }/kafka/TbKafkaDecoder.java (83%) rename common/queue/src/main/java/org/thingsboard/server/{queue => }/kafka/TbKafkaEncoder.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{queue/TbQueueHandler.java => kafka/TbKafkaHandler.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/{queue/TbQueueConsumer.java => kafka/TbKafkaPartitioner.java} (63%) rename common/queue/src/main/java/org/thingsboard/server/{queue => }/kafka/TbKafkaProperty.java (74%) rename common/queue/src/main/java/org/thingsboard/server/{queue/TbQueueMsg.java => kafka/TbKafkaRequestIdExtractor.java} (81%) rename common/queue/src/main/java/org/thingsboard/server/{queue/common/DefaultTbQueueRequestTemplate.java => kafka/TbKafkaRequestTemplate.java} (54%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java rename common/queue/src/main/java/org/thingsboard/server/{queue => }/kafka/TbKafkaSettings.java (79%) rename common/queue/src/main/java/org/thingsboard/server/{queue/sqs/TbAwsSqsSettings.java => kafka/TbNodeIdProvider.java} (50%) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java delete mode 100644 common/queue/src/main/proto/queue.proto rename common/{queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java => transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java} (77%) create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java create mode 100644 common/transport/transport-api/src/main/proto/transport.proto rename common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java => rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java (54%) delete mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java delete mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java delete mode 100644 ui-ngx/src/app/core/http/queue.service.ts delete mode 100644 ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss delete mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.html delete mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts delete mode 100644 ui-ngx/src/app/shared/models/queue.models.ts delete mode 100644 ui/src/app/api/queue.service.js delete mode 100644 ui/src/app/components/queue/index.js delete mode 100644 ui/src/app/components/queue/queue-type-list.directive.js delete mode 100644 ui/src/app/components/queue/queue-type-list.scss delete mode 100644 ui/src/app/components/queue/queue-type-list.tpl.html diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json deleted file mode 100644 index f6370696c1..0000000000 --- a/application/src/main/data/json/demo/dashboards/gateways.json +++ /dev/null @@ -1,1274 +0,0 @@ -{ - "title": "Gateways", - "configuration": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "isSystemType": true, - "bundleAlias": "entity_admin_widgets", - "typeAlias": "device_admin_table2", - "type": "latest", - "title": "New widget", - "sizeX": 24, - "sizeY": 12, - "config": { - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "4px", - "settings": { - "enableSearch": true, - "displayPagination": true, - "defaultPageSize": 10, - "defaultSortOrder": "entityName", - "displayEntityName": true, - "displayEntityType": false, - "entitiesTitle": "List of gateways", - "enableSelectColumnDisplay": true, - "displayEntityLabel": false, - "entityNameColumnTitle": "Gateway Name" - }, - "title": "Devices gateway table", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400, - "padding": "5px 10px 5px 10px" - }, - "useDashboardTimewindow": false, - "showLegend": false, - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "active", - "type": "attribute", - "label": "Active", - "color": "#2196f3", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": true, - "useCellContentFunction": true, - "cellContentFunction": "value = '⬤';\nreturn value;", - "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" - }, - "_hash": 0.3646047595211721 - }, - { - "name": "eventsSent", - "type": "timeseries", - "label": "Sent", - "color": "#4caf50", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.7235710720767985 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Events", - "color": "#f44336", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.5085933386303254 - }, - { - "name": "LOGS", - "type": "timeseries", - "label": "Latest log", - "color": "#ffc107", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3504240371585048, - "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" - }, - { - "name": "RemoteLoggingLevel", - "type": "attribute", - "label": "Log level", - "color": "#607d8b", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.9785994222542516 - } - ], - "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" - } - ], - "showTitleIcon": true, - "titleIcon": "list", - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "List device", - "widgetStyle": {}, - "displayTimewindow": true, - "actions": { - "headerButton": [ - { - "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", - "name": "Add device", - "icon": "add", - "type": "customPretty", - "customHtml": "\n \n \n
\n

Add device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n \n\n", - "customCss": "", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" - } - ], - "actionCellButton": [ - { - "id": "78845501-234e-a452-6819-82b5b776e99f", - "name": "Configuration", - "icon": "settings", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "openRightLayout": false, - "setEntityId": true - }, - { - "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", - "name": "Graphs", - "icon": "show_chart", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname_grafic", - "setEntityId": true - }, - { - "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", - "name": "Edit device", - "icon": "edit", - "type": "customPretty", - "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n \n", - "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" - }, - { - "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", - "name": "Delete device", - "icon": "delete", - "type": "custom", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" - } - ], - "rowClick": [ - { - "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", - "name": "to_entityname__config", - "icon": "more_horiz", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "setEntityId": true, - "stateEntityParamName": "" - } - ] - } - }, - "id": "94715984-ae74-76e4-20b7-2f956b01ed80" - }, - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 14, - "sizeY": 13, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "LOGS", - "type": "timeseries", - "label": "LOGS", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3496649158709739, - "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 2592000000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 10 - }, - "title": "Debug events (logs)", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" - }, - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "opcuaEventsProduced", - "type": "timeseries", - "label": "opcuaEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.1477920581839779 - }, - { - "name": "opcuaEventsSent", - "type": "timeseries", - "label": "opcuaEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6500957113784758 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 7, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "Events", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.8156044798125357 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Produced", - "color": "#4caf50", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.6538259344015449 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 604800000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 6, - "hideEmptyLines": true - }, - "title": "Total Messages", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": false, - "showMax": false, - "showAvg": true, - "showTotal": false - } - }, - "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "eventsSent", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.41414001784591314 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "eventsProduced", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.7819101846284422 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "History information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "bleEventsProduced", - "type": "timeseries", - "label": "bleEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.5625165504526104 - }, - { - "name": "bleEventsSent", - "type": "timeseries", - "label": "bleEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6817950080745288 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 5000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "isSystemType": true, - "bundleAlias": "gateway_widgets", - "typeAlias": "attributes_card", - "type": "latest", - "title": "New widget", - "sizeX": 7, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "eventsTitle": "Gateway Events Form", - "eventsReg": [ - "EventsProduced", - "EventsSent" - ] - }, - "title": "Gateway events", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "466f046d-6005-a168-b107-60fcb2469cd5" - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "isSystemType": true, - "bundleAlias": "gateway_widgets", - "typeAlias": "config_form_latest", - "type": "latest", - "title": "New widget", - "sizeX": 10, - "sizeY": 9, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "gatewayTitle": "Gateway configuration (Single device)", - "readOnly": false - }, - "title": "New Gateway configuration (Single device)", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "isSystemType": true, - "bundleAlias": "control_widgets", - "typeAlias": "rpcbutton", - "type": "rpc", - "title": "New widget", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodParams": "{}", - "methodName": "gateway_reboot", - "buttonText": "GATEWAY REBOOT" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "isSystemType": true, - "bundleAlias": "control_widgets", - "typeAlias": "rpcbutton", - "type": "rpc", - "title": "New widget", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodName": "gateway_restart", - "methodParams": "{}", - "buttonText": "gateway restart" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "isSystemType": true, - "bundleAlias": "input_widgets", - "typeAlias": "markers_placement_openstreetmap", - "type": "latest", - "title": "New widget", - "sizeX": 6, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "latitude", - "type": "attribute", - "label": "latitude", - "color": "#2196f3", - "settings": {}, - "_hash": 0.9743324774725604 - }, - { - "name": "longitude", - "type": "attribute", - "label": "longitude", - "color": "#4caf50", - "settings": {}, - "_hash": 0.5530093635101525 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": false, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "fitMapBounds": true, - "latKeyName": "latitude", - "lngKeyName": "longitude", - "showLabel": true, - "label": "${entityName}", - "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", - "markerImageSize": 34, - "useColorFunction": false, - "markerImages": [], - "useMarkerImageFunction": false, - "color": "#fe7569", - "mapProvider": "OpenStreetMap.Mapnik", - "showTooltip": true, - "autocloseTooltip": true, - "defaultCenterPosition": [ - 0, - 0 - ], - "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", - "showTooltipAction": "click", - "polygonKeyName": "coordinates", - "polygonOpacity": 0.5, - "polygonStrokeOpacity": 1, - "polygonStrokeWeight": 1, - "zoomOnClick": true, - "showCoverageOnHover": true, - "animate": true, - "maxClusterRadius": 80, - "removeOutsideVisibleBounds": true, - "defaultZoomLevel": 5 - }, - "title": "Gateway Location", - "dropShadow": true, - "enableFullscreen": false, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "widgetStyle": {}, - "actions": { - "tooltipAction": [ - { - "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", - "name": "delete", - "icon": "more_horiz", - "type": "custom", - "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" - } - ] - }, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" - } - }, - "states": { - "main_gateway": { - "name": "Gateways", - "root": true, - "layouts": { - "main": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "sizeX": 24, - "sizeY": 12, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - }, - "__entityname__config": { - "name": "${entityName} Configuration", - "root": false, - "layouts": { - "main": { - "widgets": { - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "sizeX": 14, - "sizeY": 13, - "row": 0, - "col": 10 - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "sizeX": 10, - "sizeY": 9, - "row": 0, - "col": 0 - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "sizeX": 4, - "sizeY": 2, - "row": 9, - "col": 0 - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "sizeX": 4, - "sizeY": 2, - "row": 11, - "col": 0 - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "sizeX": 6, - "sizeY": 4, - "row": 9, - "col": 4 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - }, - "__entityname_grafic": { - "name": "${entityName} Details", - "root": false, - "layouts": { - "main": { - "widgets": { - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 4, - "col": 7 - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "sizeX": 7, - "sizeY": 7, - "row": 5, - "col": 0 - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 0, - "col": 7 - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 8, - "col": 7 - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "sizeX": 7, - "sizeY": 5, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "auto 100%", - "autoFillHeight": true, - "mobileAutoFillHeight": true, - "mobileRowHeight": 70 - } - } - } - } - }, - "entityAliases": { - "3e0f533a-0db1-3292-184f-06e73535061a": { - "id": "3e0f533a-0db1-3292-184f-06e73535061a", - "alias": "Gateways", - "filter": { - "type": "deviceType", - "resolveMultiple": true, - "deviceType": "gateway", - "deviceNameFilter": "" - } - }, - "b2487e75-2fa4-f211-142c-434dfd50c70c": { - "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", - "alias": "Current Gateway", - "filter": { - "type": "stateEntity", - "resolveMultiple": false, - "stateEntityParamName": "", - "defaultStateEntity": null - } - } - }, - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "settings": { - "stateControllerId": "entity", - "showTitle": true, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": true, - "titleColor": "rgba(0,0,0,0.870588)" - } - }, - "name": "Gateways" -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json deleted file mode 100644 index b8231ab880..0000000000 --- a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json +++ /dev/null @@ -1,520 +0,0 @@ -{ - "title": "Rule Engine Statistics", - "configuration": { - "widgets": { - "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 12, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "successfulMsgs", - "type": "timeseries", - "label": "${entityName} Successful", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.15490750967648736 - }, - { - "name": "failedMsgs", - "type": "timeseries", - "label": "${entityName} Permanent Failures", - "color": "#ef5350", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.4186621166514697 - }, - { - "name": "tmpFailed", - "type": "timeseries", - "label": "${entityName} Processing Failures", - "color": "#ffc107", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.49891007198715376 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 300000 - }, - "aggregation": { - "type": "NONE", - "limit": 8640 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Queue Stats", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "actions": {}, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": true, - "showMax": true, - "showAvg": false, - "showTotal": true - } - }, - "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" - }, - "5eb79712-5c24-3060-7e4f-6af36b8f842d": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 24, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Rule Chain", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).ruleChainName;" - }, - "_hash": 0.9954481282345906 - }, - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Rule Node", - "color": "#4caf50", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).ruleNodeName;" - }, - "_hash": 0.18580357036589978 - }, - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Latest Error", - "color": "#f44336", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).message;" - }, - "_hash": 0.7255162989552142 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 10 - }, - "title": "Exceptions", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" - }, - "ad3f1417-87a8-750e-fc67-49a2de1466d4": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 12, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "timeoutMsgs", - "type": "timeseries", - "label": "${entityName} Permanent Timeouts", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.565222981550328 - }, - { - "name": "tmpTimeout", - "type": "timeseries", - "label": "${entityName} Processing Timeouts", - "color": "#9c27b0", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.2679547062508352 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 300000 - }, - "aggregation": { - "type": "NONE", - "limit": 8640 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Processing Failures and Timeouts", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "actions": {}, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": true, - "showMax": true, - "showAvg": false, - "showTotal": true - } - }, - "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" - } - }, - "states": { - "default": { - "name": "Rule Engine Statistics", - "root": true, - "layouts": { - "main": { - "widgets": { - "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { - "sizeX": 12, - "sizeY": 7, - "mobileHeight": null, - "row": 0, - "col": 0 - }, - "5eb79712-5c24-3060-7e4f-6af36b8f842d": { - "sizeX": 24, - "sizeY": 5, - "row": 7, - "col": 0 - }, - "ad3f1417-87a8-750e-fc67-49a2de1466d4": { - "sizeX": 12, - "sizeY": 7, - "mobileHeight": null, - "row": 0, - "col": 12 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - } - }, - "entityAliases": { - "140f23dd-e3a0-ed98-6189-03c49d2d8018": { - "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", - "alias": "TbServiceQueues", - "filter": { - "type": "assetType", - "resolveMultiple": true, - "assetType": "TbServiceQueue", - "assetNameFilter": "" - } - } - }, - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false, - "realtime": { - "interval": 1000, - "timewindowMs": 60000 - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1586176634823, - "endTimeMs": 1586263034823 - } - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "settings": { - "stateControllerId": "entity", - "showTitle": false, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": true - } - }, - "name": "Rule Engine Statistics" -} \ No newline at end of file diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index dda3bd7b1c..ff1fb5129b 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,28 +123,3 @@ BEGIN END LOOP; END $$; - -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e9b68882b..54ac41172c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,10 +32,12 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -43,11 +45,10 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -64,23 +65,24 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; -import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; +import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.TbCoreToTransportService; +import org.thingsboard.server.service.transport.RuleEngineTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -104,24 +106,35 @@ public class ActorSystemContext { return debugPerTenantLimits; } - @Autowired @Getter @Setter - private TbServiceInfoProvider serviceInfoProvider; + private ActorService actorService; + @Autowired @Getter - @Setter - private ActorService actorService; + private DiscoveryService discoveryService; @Autowired @Getter @Setter private ComponentDiscoveryService componentService; + @Autowired + @Getter + private ClusterRoutingService routingService; + + @Autowired + @Getter + private ClusterRpcService rpcService; + @Autowired @Getter private DataDecodingEncodingService encodingService; + @Autowired + @Getter + private DeviceAuthService deviceAuthService; + @Autowired @Getter private DeviceService deviceService; @@ -150,13 +163,6 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; - @Autowired - private PartitionService partitionService; - - @Autowired - @Getter - private TbClusterService clusterService; - @Autowired @Getter private TimeseriesService tsService; @@ -189,6 +195,10 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; + @Autowired + @Getter + private DeviceRpcService deviceRpcService; + @Autowired @Getter private JsInvokeService jsSandbox; @@ -201,6 +211,10 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; + @Autowired + @Getter + private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; + @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -217,32 +231,27 @@ public class ActorSystemContext { @Getter private MailService mailService; - //TODO: separate context for TbCore and TbRuleEngine - @Autowired(required = false) + @Autowired @Getter private DeviceStateService deviceStateService; - @Autowired(required = false) + @Autowired @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Autowired(required = false) + @Lazy + @Autowired @Getter - private TbCoreToTransportService tbCoreToTransportService; + private RuleEngineTransportService ruleEngineTransportService; - /** - * The following Service will be null if we operate in tb-core mode - */ - @Autowired(required = false) + @Lazy + @Autowired @Getter - private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; + private RuleChainTransactionService ruleChainTransactionService; - /** - * The following Service will be null if we operate in tb-rule-engine mode - */ - @Autowired(required = false) + @Value("${cluster.partition_id}") @Getter - private TbCoreDeviceRpcService tbCoreDeviceRpcService; + private long queuePartitionId; @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter @@ -260,6 +269,10 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; + @Value("${actors.client_side_rpc.timeout}") + @Getter + private long clientSideRpcTimeout; + @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; @@ -321,6 +334,11 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; + @Autowired + @Getter + private TbNodeIdProvider nodeIdProvider; + + @Getter @Setter private ActorRef appActor; @@ -347,8 +365,6 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } - - public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -358,7 +374,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); + event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); persistEvent(event); } @@ -367,7 +383,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -381,8 +397,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); + private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -392,21 +408,12 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(String serviceId, String method, String body) { - return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); + private JsonNode toBodyJson(ServerAddress server, String method, String body) { + return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - return partitionService.resolve(serviceType, tenantId, entityId); - } - - public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { - return partitionService.resolve(serviceType, queueName, tenantId, entityId); - } - - - public String getServiceId() { - return serviceInfoProvider.getServiceId(); + public String getServerAddress() { + return discoveryService.getCurrentServer().getServerAddress().toString(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { @@ -437,7 +444,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServiceId()) + .put("server", getServerAddress()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -497,7 +504,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServiceId()) + .put("server", getServerAddress()) .put("message", "Reached debug mode rate limit!"); if (error != null) { @@ -523,7 +530,4 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } - public void tell(TbActorMsg tbActorMsg, ActorRef sender) { - appActor.tell(tbActorMsg, sender); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index b7782a6a87..778c945a32 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,13 +20,19 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; +import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; +import akka.event.Logging; +import akka.event.LoggingAdapter; +import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -36,32 +42,29 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; -import java.util.HashSet; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; -import java.util.Set; -public class AppActor extends ContextAwareActor { +public class AppActor extends RuleChainManagerActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; - private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext); + super(systemContext, new SystemRuleChainManager(systemContext)); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); - this.deletedTenants = new HashSet<>(); } @Override @@ -76,7 +79,7 @@ public class AppActor extends ContextAwareActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initTenantActors(); + initRuleChainsAndTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -85,14 +88,17 @@ public class AppActor extends ContextAwareActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case PARTITION_CHANGE_MSG: + case SEND_TO_CLUSTER_MSG: + onPossibleClusterMsg((SendToClusterMsg) msg); + break; + case CLUSTER_EVENT_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case QUEUE_TO_RULE_ENGINE_MSG: - onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); + case SERVICE_TO_RULE_ENGINE_MSG: + onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -100,6 +106,7 @@ public class AppActor extends ContextAwareActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: + case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -108,30 +115,16 @@ public class AppActor extends ContextAwareActor { return true; } - private void initTenantActors() { + private void initRuleChainsAndTenantActors() { log.info("Starting main system actor."); try { - // This Service may be started for specific tenant only. - Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - if (isolatedTenantId.isPresent()) { - Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); - if (tenant != null) { + initRuleChains(); + if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + for (Tenant tenant : tenantIterator) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); - } else { - log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); - } - } else if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - for (Tenant tenant : tenantIterator) { - if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { - log.debug("[{}] Creating tenant actor", tenant.getId()); - getOrCreateTenantActor(tenant.getId()); - log.debug("[{}] Tenant actor created.", tenant.getId()); - } } } log.info("Main system actor started."); @@ -140,33 +133,40 @@ public class AppActor extends ContextAwareActor { } } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + private void onPossibleClusterMsg(SendToClusterMsg msg) { + Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); + if (address.isPresent()) { + systemContext.getRpcService().tell( + systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); + } else { + self().tell(msg.getMsg(), ActorRef.noSender()); + } + } + + private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { - msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); +// this may be a notification about system entities created. +// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); } else { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); - } else { - msg.getTbMsg().getCallback().onSuccess(); - } + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); } } + @Override protected void broadcast(Object msg) { + super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - log.warn("Message has system tenant id: {}", msg); + target = getEntityActorRef(msg.getEntityId()); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - TenantId tenantId = new TenantId(msg.getEntityId().getId()); - deletedTenants.add(tenantId); - ActorRef tenantActor = tenantActors.get(tenantId); + log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -183,22 +183,16 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); - } else { - if (msg instanceof TransportToDeviceActorMsgWrapper) { - ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); - } - } + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.info("[{}] Creating tenant actor.", tenantId); + log.debug("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 256ffd0b30..547e484652 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,8 +22,10 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -46,11 +48,6 @@ public class DeviceActor extends ContextAwareActor { } } - @Override - public void postStop() { - - } - @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { @@ -69,9 +66,15 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; + case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: + processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); + break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; + case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: + processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); + break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 437ed456e9..2752b53090 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,10 +16,13 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; +import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -35,34 +38,38 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; +import org.thingsboard.server.utils.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -93,6 +100,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; + private final Map toServerRpcPendingMap; + + private final Gson gson = new Gson(); private int rpcSeq = 0; private String deviceName; @@ -107,6 +117,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); + this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -142,7 +153,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (SessionType.SYNC == value.getType()) { + if (TransportProtos.SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -150,7 +161,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -171,16 +182,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - SessionType sessionType = getSessionType(sessionId); + TransportProtos.SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == SessionType.SYNC) { + if (sessionType == TransportProtos.SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -188,7 +199,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == SessionType.ASYNC) { + if (sessionType == TransportProtos.SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -203,7 +214,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -212,8 +223,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { + boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); - TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -223,16 +234,34 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } + if (msg.hasPostAttributes()) { + handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); + reportDeviceActivity = true; + } + if (msg.hasPostTelemetry()) { + handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); + reportDeviceActivity = true; + } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } + if (msg.hasToServerRPCCallRequest()) { + handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); + reportDeviceActivity = true; + } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - callback.onSuccess(); + if (reportDeviceActivity) { + reportLogicalDeviceActivity(); + } + } + + private void reportLogicalDeviceActivity() { + systemContext.getDeviceStateService().onDeviceActivity(deviceId); } private void reportSessionOpen() { @@ -297,8 +326,67 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; + private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { + JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), + TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + pushToRuleEngine(context, tbMsg); + } + + private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { + for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsgMetaData metaData = defaultMetaData.copy(); + metaData.putValue("ts", tsKv.getTs() + ""); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + pushToRuleEngine(context, tbMsg); + } + } + + private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { + UUID sessionId = getSessionId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", request.getMethodName()); + json.add("params", JsonUtils.parse(request.getParams())); + + TbMsgMetaData requestMetaData = defaultMetaData.copy(); + requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); + + scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); + toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); + } + + private TransportProtos.SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; + } + + void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { + ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); + if (data != null) { + log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); + sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(msg.getId()).setError("timeout").build() + , data.getSessionId(), data.getNodeId()); + } + } + + void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { + int requestId = msg.getMsg().getRequestId(); + ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); + sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() + , data.getSessionId(), data.getNodeId()); + } else { + log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); + } + } + + private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { + context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -346,7 +434,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -361,7 +449,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -382,7 +470,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -406,11 +494,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } - systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -424,10 +511,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -438,7 +525,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } - systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } @@ -450,11 +536,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -466,35 +552,35 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } - private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } @@ -546,9 +632,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - DeviceSessionsCacheEntry sessionsDump = null; + TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -557,11 +643,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); - SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); + TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -579,27 +665,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() + TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() + TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } @@ -620,5 +706,4 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } - } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java similarity index 57% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java index eb5b8e29d9..f49b2bdae8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java @@ -13,20 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.flow; +package org.thingsboard.server.actors.device; +import akka.actor.ActorRef; import lombok.Data; -import org.thingsboard.rule.engine.api.NodeConfiguration; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; +/** + * Created by ashvayka on 15.03.18. + */ @Data -public class TbCheckpointNodeConfiguration implements NodeConfiguration { +public final class DeviceActorToRuleEngineMsg implements TbActorMsg { - private String queueName; + private final ActorRef callbackRef; + private final TbMsg tbMsg; @Override - public TbCheckpointNodeConfiguration defaultConfiguration() { - TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); - configuration.setQueueName("HighPriority"); - return configuration; + public MsgType getMsgType() { + return MsgType.DEVICE_ACTOR_TO_RULE_ENGINE_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java new file mode 100644 index 0000000000..8b502b973c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.rpc.GrpcSession; +import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class BasicRpcSessionListener implements GrpcSessionListener { + + private final ClusterRpcCallbackExecutorService callbackExecutorService; + private final ActorService service; + private final ActorRef manager; + private final ActorRef self; + + BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { + this.service = context.getActorService(); + this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); + this.manager = manager; + this.self = self; + } + + @Override + public void onConnected(GrpcSession session) { + log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); + if (!session.isClient()) { + manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); + } + } + + @Override + public void onDisconnected(GrpcSession session) { + log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); + manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); + } + + @Override + public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { + log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); + callbackExecutorService.execute(() -> { + try { + service.onReceivedMsg(session.getRemoteServer(), clusterMessage); + } catch (Exception e) { + log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); + } + }); + } + + @Override + public void onError(GrpcSession session, Throwable t) { + log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); + manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); + session.close(); + } + + private static String getType(GrpcSession session) { + return session.isClient() ? "Client" : "Server"; + } + + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java index e39dc0c322..f0ff8ce9e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.actors.rpc; import lombok.Data; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineQueueSubmitStrategyConfiguration { - - private String type; - private int batchSize; - +public final class RpcBroadcastMsg { + private final ClusterAPIProtos.ClusterMessage msg; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java new file mode 100644 index 0000000000..133e7375d4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java @@ -0,0 +1,230 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import akka.actor.OneForOneStrategy; +import akka.actor.Props; +import akka.actor.SupervisorStrategy; +import akka.event.Logging; +import akka.event.LoggingAdapter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import scala.concurrent.duration.Duration; + +import java.util.*; + +/** + * @author Andrew Shvayka + */ +public class RpcManagerActor extends ContextAwareActor { + + private final Map sessionActors; + private final Map> pendingMsgs; + private final ServerAddress instance; + + private RpcManagerActor(ActorSystemContext systemContext) { + super(systemContext); + this.sessionActors = new HashMap<>(); + this.pendingMsgs = new HashMap<>(); + this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); + + systemContext.getDiscoveryService().getOtherServers().stream() + .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) + .forEach(otherServer -> onCreateSessionRequest( + new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); + } + + @Override + protected boolean process(TbActorMsg msg) { + //TODO Move everything here, to work with TbActorMsg + return false; + } + + @Override + public void onReceive(Object msg) { + if (msg instanceof ClusterAPIProtos.ClusterMessage) { + onMsg((ClusterAPIProtos.ClusterMessage) msg); + } else if (msg instanceof RpcBroadcastMsg) { + onMsg((RpcBroadcastMsg) msg); + } else if (msg instanceof RpcSessionCreateRequestMsg) { + onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); + } else if (msg instanceof RpcSessionConnectedMsg) { + onSessionConnected((RpcSessionConnectedMsg) msg); + } else if (msg instanceof RpcSessionDisconnectedMsg) { + onSessionDisconnected((RpcSessionDisconnectedMsg) msg); + } else if (msg instanceof RpcSessionClosedMsg) { + onSessionClosed((RpcSessionClosedMsg) msg); + } else if (msg instanceof ClusterEventMsg) { + onClusterEvent((ClusterEventMsg) msg); + } + } + + private void onMsg(RpcBroadcastMsg msg) { + log.debug("Forwarding msg to session actors {}", msg); + sessionActors.keySet().forEach(address -> { + ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() + .toBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(address.getHost()) + .setPort(address.getPort()) + .build()) + .build(); + onMsg(msgWithServerAddress); + }); + pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); + } + + private void onMsg(ClusterAPIProtos.ClusterMessage msg) { + if (msg.hasServerAddress()) { + ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); + SessionActorInfo session = sessionActors.get(address); + if (session != null) { + log.debug("{} Forwarding msg to session actor: {}", address, msg); + session.getActor().tell(msg, ActorRef.noSender()); + } else { + log.debug("{} Storing msg to pending queue: {}", address, msg); + Queue queue = pendingMsgs.get(address); + if (queue == null) { + queue = new LinkedList<>(); + pendingMsgs.put(new ServerAddress( + msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); + } + queue.add(msg); + } + } else { + log.warn("Cluster msg doesn't have server address [{}]", msg); + } + } + + @Override + public void postStop() { + sessionActors.clear(); + pendingMsgs.clear(); + } + + private void onClusterEvent(ClusterEventMsg msg) { + ServerAddress server = msg.getServerAddress(); + if (server.compareTo(instance) > 0) { + if (msg.isAdded()) { + onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); + } else { + onSessionClose(false, server); + } + } + } + + private void onSessionConnected(RpcSessionConnectedMsg msg) { + register(msg.getRemoteAddress(), msg.getId(), context().sender()); + } + + private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { + boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); + onSessionClose(reconnect, msg.getRemoteAddress()); + } + + private void onSessionClosed(RpcSessionClosedMsg msg) { + boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); + onSessionClose(reconnect, msg.getRemoteAddress()); + } + + private boolean isRegistered(ServerAddress address) { + for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { + if (server.getServerAddress().equals(address)) { + return true; + } + } + return false; + } + + private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { + log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); + SessionActorInfo sessionRef = sessionActors.get(remoteAddress); + if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { + context().stop(sessionRef.actor); + sessionActors.remove(remoteAddress); + pendingMsgs.remove(remoteAddress); + if (reconnect) { + onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); + } + } + } + + private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { + if (msg.getRemoteAddress() != null) { + if (!sessionActors.containsKey(msg.getRemoteAddress())) { + ActorRef actorRef = createSessionActor(msg); + register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); + } + } else { + createSessionActor(msg); + } + } + + private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { + sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); + log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); + Queue data = pendingMsgs.remove(remoteAddress); + if (data != null) { + log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); + data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); + } else { + log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); + } + } + + private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { + log.info("[{}] Creating session actor.", msg.getMsgUid()); + ActorRef actor = context().actorOf( + Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) + .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); + actor.tell(msg, context().self()); + return actor; + } + + public static class ActorCreator extends ContextBasedCreator { + private static final long serialVersionUID = 1L; + + public ActorCreator(ActorSystemContext context) { + super(context); + } + + @Override + public RpcManagerActor create() { + return new RpcManagerActor(context); + } + } + + @Override + public SupervisorStrategy supervisorStrategy() { + return strategy; + } + + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("Unknown failure", t); + return SupervisorStrategy.resume(); + }); +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java new file mode 100644 index 0000000000..af2a7f0631 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; +import org.thingsboard.server.service.cluster.rpc.GrpcSession; +import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; + +import java.util.UUID; + +import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class RpcSessionActor extends ContextAwareActor { + + + private final UUID sessionId; + private GrpcSession session; + private GrpcSessionListener listener; + + private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { + super(systemContext); + this.sessionId = sessionId; + } + + @Override + protected boolean process(TbActorMsg msg) { + //TODO Move everything here, to work with TbActorMsg + return false; + } + + @Override + public void onReceive(Object msg) { + if (msg instanceof ClusterAPIProtos.ClusterMessage) { + tell((ClusterAPIProtos.ClusterMessage) msg); + } else if (msg instanceof RpcSessionCreateRequestMsg) { + initSession((RpcSessionCreateRequestMsg) msg); + } + } + + private void tell(ClusterAPIProtos.ClusterMessage msg) { + if (session != null) { + session.sendMsg(msg); + } else { + log.trace("Failed to send message due to missing session!"); + } + } + + @Override + public void postStop() { + if (session != null) { + log.info("Closing session -> {}", session.getRemoteServer()); + try { + session.close(); + } catch (RuntimeException e) { + log.trace("Failed to close session!", e); + } + } + } + + private void initSession(RpcSessionCreateRequestMsg msg) { + log.info("[{}] Initializing session", context().self()); + ServerAddress remoteServer = msg.getRemoteAddress(); + listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); + if (msg.getRemoteAddress() == null) { + // Server session + session = new GrpcSession(listener); + session.setOutputStream(msg.getResponseObserver()); + session.initInputStream(); + session.initOutputStream(); + systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); + } else { + // Client session + ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); + session = new GrpcSession(remoteServer, listener, channel); + session.initInputStream(); + + ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); + StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); + + session.setOutputStream(outputStream); + session.initOutputStream(); + outputStream.onNext(toConnectMsg()); + } + } + + public static class ActorCreator extends ContextBasedCreator { + private static final long serialVersionUID = 1L; + + private final UUID sessionId; + + public ActorCreator(ActorSystemContext context, UUID sessionId) { + super(context); + this.sessionId = sessionId; + } + + @Override + public RpcSessionActor create() { + return new RpcSessionActor(context, sessionId); + } + } + + private ClusterAPIProtos.ClusterMessage toConnectMsg() { + ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); + return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( + ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) + .setPort(instance.getPort()).build()).build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java new file mode 100644 index 0000000000..bb87d2ea88 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionClosedMsg { + + private final boolean client; + private final ServerAddress remoteAddress; +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java new file mode 100644 index 0000000000..489a5bfa17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionConnectedMsg { + + private final ServerAddress remoteAddress; + private final UUID id; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java index 4a4829f0c2..df41a7e95f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java @@ -13,19 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.actors.rpc; +import io.grpc.stub.StreamObserver; import lombok.Data; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineProcessingDecision { +public final class RpcSessionCreateRequestMsg { - private final boolean commit; - private final ConcurrentMap> reprocessMap; + private final UUID msgUid; + private final ServerAddress remoteAddress; + private final StreamObserver responseObserver; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java new file mode 100644 index 0000000000..14e7504637 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionDisconnectedMsg { + + private final boolean client; + private final ServerAddress remoteAddress; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java index 0d21c59c9c..3832d6eb94 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java @@ -13,16 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.actors.rpc; import lombok.Data; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineQueueAckStrategyConfiguration { - - private String type; - private int retries; - private double failurePercentage; - private long pauseBetweenRetries; - +public final class RpcSessionTellMsg { + private final ClusterAPIProtos.ClusterMessage msg; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java new file mode 100644 index 0000000000..811713819a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import lombok.Data; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +public final class SessionActorInfo { + protected final UUID sessionId; + protected final ActorRef actor; +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 6a3994a36f..b700007498 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,13 +17,18 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; -import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -35,15 +40,19 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -58,13 +67,11 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -72,7 +79,6 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ -@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -85,11 +91,6 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } - @Override - public void tellSuccess(TbMsg msg) { - tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); - } - @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); @@ -100,11 +101,16 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } + @Override + public void tellNext(TbMsg msg, String relationType, Throwable th) { + tellNext(msg, Collections.singleton(relationType), th); + } + private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); } @Override @@ -113,94 +119,10 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } - @Override - public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueue(tpi, tbMsg, onFailure, onSuccess); - } - - @Override - public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueue(tpi, tbMsg, onFailure, onSuccess); - } - - private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); - } - - @Override - public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String relationType) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); - } - - private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { - RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); - RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); - tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); - TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)) - .addAllRelationTypes(relationTypes); - if (failureMessage != null) { - msg.setFailureMessage(failureMessage); - } - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); - } - - @Override - public void ack(TbMsg tbMsg) { - if (nodeCtx.getSelf().isDebugMode()) { - mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); - } - tbMsg.getCallback().onSuccess(); - } - @Override public boolean isLocalEntity(EntityId entityId) { - return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); + Optional address = mainCtx.getRoutingService().resolveById(entityId); + return !address.isPresent(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -212,48 +134,66 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), - msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); } + @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return TbMsg.transformMsg(origMsg, type, originator, metaData, data); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); + } + + @Override + public void sendTbMsgToRuleEngine(TbMsg msg) { + mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - return entityCreatedMsg(customer, customer.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(customer); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process customer created msg: " + e); + } } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - return entityCreatedMsg(device, device.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(device); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process device created msg: " + e); + } } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - return entityCreatedMsg(asset, asset.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(asset); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process asset created msg: " + e); + } } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { - return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); - } - - public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); + ObjectNode entityNode = mapper.valueToTree(alarm); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); + throw new RuntimeException("Failed to process alarm created msg: " + e); } } + @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -311,8 +251,8 @@ class DefaultTbContext implements TbContext { } @Override - public String getServiceId() { - return mainCtx.getServiceInfoProvider().getServiceId(); + public String getNodeId() { + return mainCtx.getNodeIdProvider().getNodeId(); } @Override @@ -380,6 +320,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } + @Override + public RuleChainTransactionService getRuleChainTransactionService() { + return mainCtx.getRuleChainTransactionService(); + } + @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); @@ -396,7 +341,35 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return mainCtx.getTbRuleEngineDeviceRpcService(); + return new RuleEngineRpcService() { + @Override + public void sendRpcReply(DeviceId deviceId, int requestId, String body) { + mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); + } + + @Override + public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + ServerAddress requestOriginAddress; + if (!StringUtils.isEmpty(src.getOriginHost())) { + requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); + } else { + requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); + } + mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + }; } @Override @@ -414,6 +387,10 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } + @Override + public String getServerAddress() { + return mainCtx.getServerAddress(); + } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -421,29 +398,4 @@ class DefaultTbContext implements TbContext { return metaData; } - private class SimpleTbQueueCallback implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } else { - log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); - } - } - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java new file mode 100644 index 0000000000..d062c5dc49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.ruleChain; + +import lombok.Data; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; + +import java.io.Serializable; + +/** + * Created by ashvayka on 19.03.18. + */ +@Data +final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { + + private static final long serialVersionUID = 2459605482321657447L; + private final TenantId tenantId; + private final RuleChainId ruleChainId; + + public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { + super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); + this.tenantId = tenantId; + this.ruleChainId = ruleChainId; + } + + @Override + public MsgType getMsgType() { + return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 027f4aac9d..72faaef533 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.actors.ruleChain; +import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { - super(systemContext, tenantId, ruleChain.getId()); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { + super(systemContext, tenantId, ruleChainId); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, context().parent(), context().self())); } @@ -43,17 +43,20 @@ public class RuleChainActor extends ComponentActor { + private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final TbClusterService clusterService; - private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; + private String ruleChainName; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChain.getId()); - this.ruleChainName = ruleChain.getName(); + super(systemContext, tenantId, ruleChainId); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.clusterService = systemContext.getClusterService(); + this.ruleChainName = ruleChainId.toString(); } @Override @@ -96,6 +92,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -155,8 +152,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); + public void onClusterEventMsg(ClusterEventMsg msg) { + } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { @@ -195,123 +192,100 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes, String failureMessage) { - try { - checkActive(); - EntityId entityId = msg.getOriginator(); - TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(relationTypes, r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (relationTypes.contains(TbRelationTypes.FAILURE)) { - RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); - if (ruleNodeCtx != null) { - msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); - } else { - log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); - msg.getCallback().onFailure(new RuleEngineException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); - } - } else { - msg.getCallback().onSuccess(); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(tpi, msg, relation.getOut(), relation.getType()); - } + void onRuleChainToRuleChainMsg(RuleChainToRuleChainMsg envelope) { + checkActive(); + if (envelope.isEnqueue()) { + if (firstNode != null) { + pushMsgToNode(firstNode, enrichWithRuleChainId(envelope.getMsg()), envelope.getFromRelationType()); + } + } else { + if (firstNode != null) { + pushMsgToNode(firstNode, envelope.getMsg(), envelope.getFromRelationType()); } else { - MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - putToQueue(tpi, msg, callbackWrapper, target); - } +// TODO: Ack this message in Kafka +// TbMsg msg = envelope.getMsg(); +// EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); +// queue.ack(tenantId, envelope.getMsg(), ackId.getId(), msg.getClusterPartition()); } - } catch (Exception e) { - msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage())); } } - private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { - switch (target.getEntityType()) { - case RULE_NODE: - putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); - break; - case RULE_CHAIN: - putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); - break; + void onTellNext(RuleNodeToRuleChainTellNextMsg envelope) { + checkActive(); + TbMsg msg = envelope.getMsg(); + EntityId originatorEntityId = msg.getOriginator(); + Optional address = systemContext.getRoutingService().resolveById(originatorEntityId); + + if (address.isPresent()) { + onRemoteTellNext(address.get(), envelope); + } else { + onLocalTellNext(envelope); } } - private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { - if (tpi.isMyPartition()) { - switch (target.getEntityType()) { - case RULE_NODE: - pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); - break; - case RULE_CHAIN: - parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); - break; + private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { + TbMsg msg = envelope.getMsg(); + log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); + envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); + systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + } + + private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { + TbMsg msg = envelope.getMsg(); + RuleNodeId originatorNodeId = envelope.getOriginator(); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + if (ackId != null) { +// TODO: Ack this message in Kafka +// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(msg, relation.getOut(), relation.getType()); } } else { - putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + switch (target.getEntityType()) { + case RULE_NODE: + enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); + break; + case RULE_CHAIN: + enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); + break; + } + } + //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. + if (ackId != null) { +// TODO: Ack this message in Kafka +// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); + } } } - private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { - ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(newMsg)) - .build(); - clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); - } - private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -324,13 +298,38 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; + protected final RuleChainManager ruleChainManager; + protected final RuleChainService ruleChainService; - public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { super(systemContext); - this.tenantId = tenantId; - this.actors = HashBiMap.create(); + this.ruleChainManager = ruleChainManager; this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { - RuleChainId ruleChainId = ruleChain.getId(); - log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); - visit(ruleChain, actorRef); - log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); - } - } - - protected void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - - public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { - return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); - } - - public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { - return actors.computeIfAbsent(ruleChainId, eId -> { - RuleChain ruleChain = provider.apply(eId); - return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) - .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); - }); + ruleChainManager.init(this.context()); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - if (entityId.getEntityType() == EntityType.RULE_CHAIN) { - target = getOrCreateActor(this.context(), (RuleChainId) entityId); + switch (entityId.getEntityType()) { + case RULE_CHAIN: + target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); + break; } return target; } protected void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); + ruleChainManager.broadcast(msg); } - - public ActorRef get(RuleChainId id) { - return actors.get(id); - } - } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java index 0122471c52..e081768d93 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java @@ -32,6 +32,7 @@ public final class RuleChainToRuleChainMsg implements TbActorMsg, RuleChainAware private final RuleChainId source; private final TbMsg msg; private final String fromRelationType; + private final boolean enqueue; @Override public RuleChainId getRuleChainId() { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index abb98468bf..65227818e8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -54,9 +53,6 @@ public class RuleNodeActor extends ComponentActor relationTypes; private final TbMsg msg; - private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index f6403c5d56..3032b1dfe0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -15,7 +15,24 @@ */ package org.thingsboard.server.actors.service; -public interface ActorService { +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.transport.SessionMsgProcessor; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; +import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; +public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + + void onMsg(SendToClusterMsg msg); + + void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); + + void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java index 2a92caf4f6..3260c7ee31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.actors.stats.StatsPersistMsg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; /** @@ -115,9 +115,9 @@ public abstract class ComponentActor status = system.terminate(); @@ -97,4 +132,157 @@ public class DefaultActorService implements ActorService { } } + @Override + public void onMsg(SendToClusterMsg msg) { + appActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onServerAdded(ServerInstance server) { + log.trace("Processing onServerAdded msg: {}", server); + broadcast(new ClusterEventMsg(server.getServerAddress(), true)); + } + + @Override + public void onServerUpdated(ServerInstance server) { + //Do nothing + } + + @Override + public void onServerRemoved(ServerInstance server) { + log.trace("Processing onServerRemoved msg: {}", server); + broadcast(new ClusterEventMsg(server.getServerAddress(), false)); + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + @Override + public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { + DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); + appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); + } + + @Override + public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { + log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); + DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); + appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); + } + + public void broadcast(ToAllNodesMsg msg) { + actorContext.getEncodingService().encode(msg); + rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage + .newBuilder() + .setPayload(ByteString + .copyFrom(actorContext.getEncodingService().encode(msg))) + .setMessageType(CLUSTER_ACTOR_MESSAGE) + .build())); + appActor.tell(msg, ActorRef.noSender()); + } + + private void broadcast(ClusterEventMsg msg) { + this.appActor.tell(msg, ActorRef.noSender()); + this.rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); + private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); + + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int sent = sentClusterMsgs.getAndSet(0); + int received = receivedClusterMsgs.getAndSet(0); + if (sent > 0 || received > 0) { + log.info("Cluster msgs sent [{}] received [{}]", sent, received); + } + } + } + + @Override + public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { + if (statsEnabled) { + receivedClusterMsgs.incrementAndGet(); + } + ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); + if (log.isDebugEnabled()) { + log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); + log.info("MSG: {}", msg); + } + switch (msg.getMessageType()) { + case CLUSTER_ACTOR_MESSAGE: + java.util.Optional decodedMsg = actorContext.getEncodingService() + .decode(msg.getPayload().toByteArray()); + if (decodedMsg.isPresent()) { + appActor.tell(decodedMsg.get(), ActorRef.noSender()); + } else { + log.error("Error during decoding cluster proto message"); + } + break; + case TO_ALL_NODES_MSG: + //TODO + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: + actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: + actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: + actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: + actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: + actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TRANSACTION_SERVICE_MESSAGE: + actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); + break; + } + } + + @Override + public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { + if (statsEnabled) { + sentClusterMsgs.incrementAndGet(); + } + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { + if (statsEnabled) { + sentClusterMsgs.incrementAndGet(); + } + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onBroadcastMsg(RpcBroadcastMsg msg) { + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onDeviceAdded(Device device) { + deviceStateService.onDeviceAdded(device); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 1443c61e16..2fdc918f8e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,13 +16,20 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; +import akka.event.LoggingAdapter; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; + +import javax.annotation.Nullable; +import java.util.function.Consumer; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -43,7 +50,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; + public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java new file mode 100644 index 0000000000..b824cf61ff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared; + +import akka.actor.ActorContext; +import akka.actor.ActorRef; +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.SearchTextBased; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.page.PageDataIterable; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public abstract class EntityActorsManager> { + + protected final ActorSystemContext systemContext; + protected final BiMap actors; + + public EntityActorsManager(ActorSystemContext systemContext) { + this.systemContext = systemContext; + this.actors = HashBiMap.create(); + } + + protected abstract TenantId getTenantId(); + + protected abstract String getDispatcherName(); + + protected abstract Creator creator(T entityId); + + protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); + + public void init(ActorContext context) { + for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { + T entityId = (T) entity.getId(); + log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(context, entityId); + visit(entity, actorRef); + log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); + } + } + + public void visit(M entity, ActorRef actorRef) { + } + + public ActorRef getOrCreateActor(ActorContext context, T entityId) { + return actors.computeIfAbsent(entityId, eId -> + context.actorOf(Props.create(creator(eId)) + .withDispatcher(getDispatcherName()), eId.toString())); + } + + public void broadcast(Object msg) { + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); + } + + public void remove(T id) { + actors.remove(id); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java new file mode 100644 index 0000000000..ef2b4282d4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import akka.actor.ActorRef; +import akka.japi.Creator; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.ruleChain.RuleChainActor; +import org.thingsboard.server.actors.shared.EntityActorsManager; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.rule.RuleChainService; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public abstract class RuleChainManager extends EntityActorsManager { + + protected final RuleChainService service; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; + + public RuleChainManager(ActorSystemContext systemContext) { + super(systemContext); + this.service = systemContext.getRuleChainService(); + } + + @Override + public Creator creator(RuleChainId entityId) { + return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); + } + + @Override + public void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java new file mode 100644 index 0000000000..10005b040b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.model.ModelConstants; + +import java.util.Collections; + +public class SystemRuleChainManager extends RuleChainManager { + + public SystemRuleChainManager(ActorSystemContext systemContext) { + super(systemContext); + } + + @Override + protected FetchFunction getFetchEntitiesFunction() { + return link -> new PageData<>(); + } + + @Override + protected TenantId getTenantId() { + return ModelConstants.SYSTEM_TENANT; + } + + @Override + protected String getDispatcherName() { + return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java new file mode 100644 index 0000000000..cfe862af49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import akka.actor.ActorContext; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; +import org.thingsboard.server.common.data.rule.RuleChain; + +public class TenantRuleChainManager extends RuleChainManager { + + private final TenantId tenantId; + + public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.tenantId = tenantId; + } + + @Override + public void init(ActorContext context) { + super.init(context); + } + + @Override + protected TenantId getTenantId() { + return tenantId; + } + + @Override + protected String getDispatcherName() { + return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; + } + + @Override + protected FetchFunction getFetchEntitiesFunction() { + return link -> service.findTenantRuleChains(tenantId, link); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 717dcecc19..31461a8fac 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { @@ -57,12 +58,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 3cee82127c..4aacfb55bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,43 +22,41 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; +import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.device.DeviceActorCreator; +import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import scala.concurrent.duration.Duration; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; +import java.util.HashMap; +import java.util.Map; public class TenantActor extends RuleChainManagerActor { + private final TenantId tenantId; private final BiMap deviceActors; - private boolean isRuleEngineForCurrentTenant; - private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, tenantId); + super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); + this.tenantId = tenantId; this.deviceActors = HashBiMap.create(); } @@ -67,37 +65,12 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } - boolean cantFindTenant = false; - @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); - if (tenant == null) { - cantFindTenant = true; - log.info("[{}] Started tenant actor for missing tenant.", tenantId); - } else { - // This Service may be started for specific tenant only. - Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - - if (isRuleEngineForCurrentTenant) { - try { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - log.info("[{}] Going to init rule chains", tenantId); - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; - } - } catch (Exception e) { - cantFindTenant = true; - } - } - log.info("[{}] Tenant actor started.", tenantId); - } + initRuleChains(); + log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -110,36 +83,18 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { - if (cantFindTenant) { - log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); - if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { - QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; - queueMsg.getTbMsg().getCallback().onSuccess(); - } - return true; - } switch (msg.getMsgType()) { - case PARTITION_CHANGE_MSG: - PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; - ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType(); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - //To Rule Chain Actors - broadcast(msg); - } else if (ServiceType.TB_CORE.equals(serviceType)) { - //To Device Actors - List repartitionedDevices = - deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); - for (DeviceId deviceId : repartitionedDevices) { - ActorRef deviceActor = deviceActors.remove(deviceId); - context().stop(deviceActor); - } - } + case CLUSTER_EVENT_MSG: + broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case QUEUE_TO_RULE_ENGINE_MSG: - onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); + case SERVICE_TO_RULE_ENGINE_MSG: + onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + break; + case DEVICE_ACTOR_TO_RULE_ENGINE_MSG: + onDeviceActorToRuleEngineMsg((DeviceActorToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -150,6 +105,7 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: + case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -158,59 +114,41 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private boolean isMyPartition(DeviceId deviceId) { - return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); + private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + if (ruleChainManager.getRootChainActor() != null) { + ruleChainManager.getRootChainActor().tell(msg, self()); + } else { + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { - if (!isRuleEngineForCurrentTenant) { - log.warn("RECEIVED INVALID MESSAGE: {}", msg); - return; - } - TbMsg tbMsg = msg.getTbMsg(); - if (tbMsg.getRuleChainId() == null) { - if (getRootChainActor() != null) { - getRootChainActor().tell(msg, self()); - } else { - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); - log.info("[{}] No Root Chain: {}", tenantId, msg); - } + private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) { + if (ruleChainManager.getRootChainActor() != null) { + ruleChainManager.getRootChainActor().tell(msg, self()); } else { - ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); - if (ruleChainActor != null) { - ruleChainActor.tell(msg, self()); - } else { - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); - //TODO: 3.1 Log it to dead letters queue; - tbMsg.getCallback().onSuccess(); - } + log.info("[{}] No Root Chain: {}", tenantId, msg); } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { - if (!isCore) { - log.warn("RECEIVED INVALID MESSAGE: {}", msg); - } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - if (isRuleEngineForCurrentTenant) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - visit(ruleChain, target); - } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + ruleChainManager.visit(ruleChain, target); } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -234,7 +172,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.debug("Removed actor was not found in the device map!"); + log.warn("[{}] Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); @@ -257,12 +195,15 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { + @Override + public SupervisorStrategy.Directive apply(Throwable t) { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); + } } }); diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index e57695da69..5e5bd94afa 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,13 +32,11 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration -@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { @@ -64,8 +62,7 @@ public class WebSocketConfiguration implements WebSocketConfigurer { SecurityUser user = null; try { user = getCurrentUser(); - } catch (ThingsboardException ex) { - } + } catch (ThingsboardException ex) {} if (user == null) { response.setStatusCode(HttpStatus.UNAUTHORIZED); return false; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 8873d82366..b7efc180a5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,25 +25,23 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; +import org.thingsboard.server.common.data.UpdateMessage; @RestController -@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { @Autowired private MailService mailService; - + @Autowired private AdminSettingsService adminSettingsService; @@ -67,7 +65,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/settings", method = RequestMethod.POST) - @ResponseBody + @ResponseBody public AdminSettings saveAdminSettings(@RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -113,8 +111,8 @@ public class AdminController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); adminSettings = checkNotNull(adminSettings); if (adminSettings.getKey().equals("mail")) { - String email = getCurrentUser().getEmail(); - mailService.sendTestMail(adminSettings.getJsonValue(), email); + String email = getCurrentUser().getEmail(); + mailService.sendTestMail(adminSettings.getJsonValue(), email); } } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 85ee63b3b1..9999ab6709 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -41,14 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.UUID; @RestController -@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 6963a35d6f..87a5ab3fc4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -33,15 +33,16 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -51,7 +52,6 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 7eb74fef90..8612d9b027 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -40,7 +39,6 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 42da043a91..44449eb3d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,7 +40,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -57,7 +56,6 @@ import java.net.URI; import java.net.URISyntaxException; @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 36bc2b104c..02e0cef53c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -26,27 +27,16 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.DashboardInfo; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceInfo; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -74,6 +64,8 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -94,11 +86,7 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -117,7 +105,6 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j -@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -168,7 +155,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected TbClusterService tbClusterService; + protected ActorService actorService; @Autowired protected RelationService relationService; @@ -191,12 +178,6 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; - @Autowired - protected PartitionService partitionService; - - @Autowired - protected TbQueueProducerProvider producerProvider; - @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -713,14 +694,10 @@ public abstract class BaseController { } } } - TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - TenantId tenantId = user.getTenantId(); - if (tenantId.isNullUid()) { - if (entity instanceof HasTenantId) { - tenantId = ((HasTenantId) entity).getTenantId(); - } - } - tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index d75da8b34c..ef9508ad7c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,14 +25,12 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController -@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 5d7e662a72..65c031172d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,12 +36,10 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 1b972a5319..a0b3904d7a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,7 +40,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -47,7 +47,6 @@ import java.util.HashSet; import java.util.Set; @RestController -@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index abf00e2bca..c2e82f897e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -31,8 +31,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; @@ -53,7 +51,6 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -65,7 +62,6 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { @@ -113,8 +109,12 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), - savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); + actorService + .onDeviceNameOrTypeUpdate( + savedDevice.getTenantId(), + savedDevice.getId(), + savedDevice.getName(), + savedDevice.getType()); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -267,9 +267,7 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - - tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); - + actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 88aa9ce850..8058cca685 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @@ -41,7 +40,6 @@ import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 1611e71c27..67c8ea0688 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -62,7 +61,6 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index cf868b29a2..cba2b2bcc0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,11 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController -@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java deleted file mode 100644 index a0b46f9166..0000000000 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -@RestController -@TbCoreComponent -@RequestMapping("/api") -public class QueueController extends BaseController { - - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) - @ResponseBody - public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { - checkParameter("serviceType", serviceType); - try { - ServiceType type = ServiceType.valueOf(serviceType); - switch (type) { - case TB_RULE_ENGINE: - return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); - default: - return Collections.emptyList(); - } - } catch (Exception e) { - throw handleException(e); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index b88c3f3695..a8298ce82b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -31,6 +31,7 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -42,25 +43,27 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. */ @RestController -@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { @@ -69,7 +72,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private TbCoreDeviceRpcService deviceRpcService; + private DeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -88,6 +91,7 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } + private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); @@ -112,7 +116,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index a97b976bb7..cc6d4447e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -52,10 +52,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -69,7 +67,6 @@ import java.util.stream.Collectors; @Slf4j @RestController -@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { @@ -133,7 +130,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -164,7 +161,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -172,7 +169,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -206,7 +203,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -258,9 +255,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, @@ -321,7 +318,7 @@ public class RuleChainController extends BaseController { ScriptEngine engine = null; try { engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); - TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); + TbMsg inMsg = new TbMsg(UUIDs.timeBased(), msgType, null, new TbMsgMetaData(metadata), data, null, null, 0L); switch (scriptType) { case "update": output = msgToOutput(engine.executeUpdate(inMsg)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index bb866e3e9b..f607d57cf6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -69,9 +69,9 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -99,7 +99,6 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController -@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { @@ -344,7 +343,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -354,18 +353,19 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { + if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify), null); + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -397,6 +397,12 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); + if (entityId.getEntityType() == EntityType.DEVICE) { + DeviceId deviceId = new DeviceId(entityId.getId()); + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 18d3e80222..f9781269d7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,13 +34,11 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { @@ -76,6 +74,7 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); + tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -95,7 +94,8 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + + actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 7f190f93cd..c93d672e07 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -56,7 +55,6 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController -@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index debe49b018..ce3e65c0a1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,14 +31,12 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index c0cf672c6f..b13b3aea54 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,14 +32,13 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 6e33133522..f2862c5ab6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -53,7 +52,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service -@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { @@ -65,6 +63,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.send_timeout:5000}") private long sendTimeout; + @Value("${server.ws.limits.max_sessions_per_tenant:0}") private int maxSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer:0}") @@ -92,10 +91,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.trace("[{}] Failed to find session", session.getId()); + log.warn("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -139,7 +138,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.trace("[{}] Failed to find session", session.getId()); + log.warn("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 7137155ad5..e0daf43bf7 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.DatabaseTsUpgradeService; +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; @@ -152,11 +152,6 @@ public class ThingsboardInstallService { log.info("Updating system data..."); - systemDataLoaderService.updateSystemWidgets(); - break; - case "2.5.0": - log.info("Upgrading ThingsBoard from version 2.5 to 3.0 ..."); - log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java new file mode 100644 index 0000000000..9a7c83f167 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; + +import javax.annotation.PostConstruct; + +import static org.thingsboard.server.utils.MiscUtils.missingProperty; + +/** + * @author Andrew Shvayka + */ +@Service +@Slf4j +public class CurrentServerInstanceService implements ServerInstanceService { + + @Value("${rpc.bind_host}") + private String rpcHost; + @Value("${rpc.bind_port}") + private Integer rpcPort; + + private ServerInstance self; + + @PostConstruct + public void init() { + Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); + Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); + self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); + log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); + } + + @Override + public ServerInstance getSelf() { + return self; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java index 36510261d6..d0ba27814c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java @@ -13,8 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; +import java.util.List; + +/** + * @author Andrew Shvayka + */ public interface DiscoveryService { + void publishCurrentServer(); + + void unpublishCurrentServer(); + + ServerInstance getCurrentServer(); + + List getOtherServers(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java new file mode 100644 index 0000000000..ef6b565817 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +/** + * @author Andrew Shvayka + */ +public interface DiscoveryServiceListener { + + void onServerAdded(ServerInstance server); + + void onServerUpdated(ServerInstance server); + + void onServerRemoved(ServerInstance server); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java similarity index 58% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java index 9017823efb..009a7801e7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java @@ -13,35 +13,55 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.DependsOn; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import java.util.Collections; +import java.util.List; +/** + * @author Andrew Shvayka + */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") public class DummyDiscoveryService implements DiscoveryService { - private final TbServiceInfoProvider serviceInfoProvider; - private final PartitionService partitionService; + @Autowired + private ServerInstanceService serverInstance; + + @PostConstruct + public void init() { + log.info("Initializing..."); + } + + @Override + public void publishCurrentServer() { + //Do nothing + } + @Override + public void unpublishCurrentServer() { + //Do nothing + } - public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; + @Override + public ServerInstance getCurrentServer() { + return serverInstance.getSelf(); } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); + @Override + public List getOtherServers() { + return Collections.emptyList(); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java new file mode 100644 index 0000000000..e9325800af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@ToString +@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) +public final class ServerInstance implements Comparable { + + @Getter + private final String host; + @Getter + private final int port; + @Getter + private final ServerAddress serverAddress; + + public ServerInstance(ServerAddress serverAddress) { + this.serverAddress = serverAddress; + this.host = serverAddress.getHost(); + this.port = serverAddress.getPort(); + } + + @Override + public int compareTo(ServerInstance o) { + return this.serverAddress.compareTo(o.serverAddress); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java index d23cc3581d..095e10f0b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.subscription; +package org.thingsboard.server.service.cluster.discovery; -public enum TbAttributeSubscriptionScope { - - CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE +/** + * @author Andrew Shvayka + */ +public interface ServerInstanceService { + ServerInstance getSelf(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java similarity index 67% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java index e761a229c4..e1cdd5f83e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.SerializationException; +import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -30,14 +32,22 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; +import org.thingsboard.server.utils.MiscUtils; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -49,6 +59,9 @@ import java.util.stream.Collectors; import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; +/** + * @author Andrew Shvayka + */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j @@ -65,29 +78,42 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @Value("${zk.zk_dir}") private String zkDir; - private final TbServiceInfoProvider serviceInfoProvider; - private final PartitionService partitionService; + private String zkNodesDir; + + @Autowired + private ServerInstanceService serverInstance; + + @Autowired + @Lazy + private TelemetrySubscriptionService tsSubService; + + @Autowired + @Lazy + private DeviceStateService deviceStateService; + + @Autowired + @Lazy + private ActorService actorService; + + @Autowired + @Lazy + private ClusterRoutingService routingService; private ExecutorService reconnectExecutorService; + private CuratorFramework client; private PathChildrenCache cache; private String nodePath; - private String zkNodesDir; private volatile boolean stopped = true; - public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; - } - @PostConstruct public void init() { log.info("Initializing..."); - Assert.hasLength(zkUrl, missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); + Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); @@ -97,47 +123,53 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi initZkClient(); } - private List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return TransportProtos.ServiceInfo.parseFrom(cd.getData()); - } catch (NoSuchElementException | InvalidProtocolBufferException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } else { - log.info("Received application ready event. Starting current ZK node."); - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) {} + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); + } + + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); } + @Override public synchronized void publishCurrentServer() { - TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + ServerInstance self = this.serverInstance.getSelf(); if (currentServerExists()) { - log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); + log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); } else { try { - log.info("[{}] Creating ZK node for current instance", self.getServiceId()); + log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); - log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); + log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -151,10 +183,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } try { - TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); - TransportProtos.ServiceInfo registeredServerInfo = null; - registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); - if (self.equals(registeredServerInfo)) { + ServerInstance self = this.serverInstance.getSelf(); + ServerAddress registeredServerAdress = null; + registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); + if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -165,9 +197,9 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } - private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { + private ConnectionStateListener checkReconnect(ServerInstance self) { return (client, newState) -> { - log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); + log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -191,25 +223,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void unpublishCurrentServer() { + @Override + public void unpublishCurrentServer() { try { if (nodePath != null) { client.delete().forPath(nodePath); @@ -220,26 +235,41 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); + @Override + public ServerInstance getCurrentServer() { + return serverInstance.getSelf(); } - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); + @Override + public List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); + } catch (NoSuchElementException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); } - public static String missingProperty(String propertyName) { - return "The " + propertyName + " property need to be set!"; + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting current ZK node."); + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + getOtherServers().forEach( + server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) + ); } @Override @@ -267,23 +297,34 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); return; } - TransportProtos.ServiceInfo instance; + ServerInstance instance; try { - instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); - } catch (InvalidProtocolBufferException e) { + ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); + instance = new ServerInstance(serverAddress); + } catch (SerializationException e) { log.error("Failed to decode server instance for node {}", data.getPath(), e); throw e; } - log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); switch (pathChildrenCacheEvent.getType()) { case CHILD_ADDED: + routingService.onServerAdded(instance); + tsSubService.onClusterUpdate(); + deviceStateService.onClusterUpdate(); + actorService.onServerAdded(instance); + break; case CHILD_UPDATED: + routingService.onServerUpdated(instance); + actorService.onServerUpdated(instance); + break; case CHILD_REMOVED: - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + routingService.onServerRemoved(instance); + tsSubService.onClusterUpdate(); + deviceStateService.onClusterUpdate(); + actorService.onServerRemoved(instance); break; default: break; } } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java rename to application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java index fcc7c7a60d..e0ed64fdbd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java @@ -13,22 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.routing; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; import java.util.Optional; +import java.util.UUID; -public interface TbServiceInfoProvider { - - String getServiceId(); - - ServiceInfo getServiceInfo(); +/** + * @author Andrew Shvayka + */ +public interface ClusterRoutingService extends DiscoveryServiceListener { - boolean isService(ServiceType serviceType); + ServerAddress getCurrentServer(); - Optional getIsolatedTenant(); + Optional resolveById(EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java new file mode 100644 index 0000000000..e114a9b6ab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java @@ -0,0 +1,153 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.utils.MiscUtils; + +import javax.annotation.PostConstruct; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Cluster service implementation based on consistent hash ring + */ + +@Service +@Slf4j +public class ConsistentClusterRoutingService implements ClusterRoutingService { + + @Autowired + private DiscoveryService discoveryService; + + @Value("${cluster.hash_function_name}") + private String hashFunctionName; + @Value("${cluster.vitrual_nodes_size}") + private Integer virtualNodesSize; + + private ServerInstance currentServer; + + private HashFunction hashFunction; + + private ConsistentHashCircle[] circles; + private ConsistentHashCircle rootCircle; + + @PostConstruct + public void init() { + log.info("Initializing Cluster routing service!"); + this.hashFunction = MiscUtils.forName(hashFunctionName); + this.currentServer = discoveryService.getCurrentServer(); + this.circles = new ConsistentHashCircle[ServerType.values().length]; + for (ServerType serverType : ServerType.values()) { + circles[serverType.ordinal()] = new ConsistentHashCircle(); + } + rootCircle = circles[ServerType.CORE.ordinal()]; + addNode(discoveryService.getCurrentServer()); + for (ServerInstance instance : discoveryService.getOtherServers()) { + addNode(instance); + } + logCircle(); + log.info("Cluster routing service initialized!"); + } + + @Override + public ServerAddress getCurrentServer() { + return discoveryService.getCurrentServer().getServerAddress(); + } + + @Override + public Optional resolveById(EntityId entityId) { + return resolveByUuid(rootCircle, entityId.getId()); + } + + private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { + Assert.notNull(uuid); + if (circle.isEmpty()) { + return Optional.empty(); + } + Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) + .putLong(uuid.getLeastSignificantBits()).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = + circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + ServerInstance result = circle.get(hash); + if (!currentServer.equals(result)) { + return Optional.of(result.getServerAddress()); + } else { + return Optional.empty(); + } + } + + @Override + public void onServerAdded(ServerInstance server) { + log.info("On server added event: {}", server); + addNode(server); + logCircle(); + } + + @Override + public void onServerUpdated(ServerInstance server) { + log.debug("Ignoring server onUpdate event: {}", server); + } + + @Override + public void onServerRemoved(ServerInstance server) { + log.info("On server removed event: {}", server); + removeNode(server); + logCircle(); + } + + private void addNode(ServerInstance instance) { + for (int i = 0; i < virtualNodesSize; i++) { + circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); + } + } + + private void removeNode(ServerInstance instance) { + for (int i = 0; i < virtualNodesSize; i++) { + circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); + } + } + + private HashCode hash(ServerInstance instance, int i) { + return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); + } + + private void logCircle() { + log.trace("Consistent Hash Circle Start"); + Arrays.asList(circles).forEach(ConsistentHashCircle::log); + log.trace("Consistent Hash Circle End"); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java similarity index 69% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java rename to application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java index 1b40545cde..711677dc78 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.routing; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -24,10 +25,11 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = + new ConcurrentSkipListMap<>(); - public void put(long hash, T instance) { + public void put(long hash, ServerInstance instance) { circle.put(hash, instance); } @@ -43,7 +45,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -51,11 +53,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public T get(Long hash) { + public ServerInstance get(Long hash) { return circle.get(hash); } public void log() { - circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); + circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java new file mode 100644 index 0000000000..23b050634c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import com.google.protobuf.ByteString; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Andrew Shvayka + */ +@Service +@Slf4j +public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { + + @Autowired + private ServerInstanceService instanceService; + + @Autowired + private DataDecodingEncodingService encodingService; + + private RpcMsgListener listener; + + private Server server; + + private ServerInstance instance; + + private ConcurrentMap>> pendingSessionMap = + new ConcurrentHashMap<>(); + + public void init(RpcMsgListener listener) { + this.listener = listener; + log.info("Initializing RPC service!"); + instance = instanceService.getSelf(); + server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); + log.info("Going to start RPC server using port: {}", instance.getPort()); + try { + server.start(); + } catch (IOException e) { + log.error("Failed to start RPC server!", e); + throw new RuntimeException("Failed to start RPC server!"); + } + log.info("RPC service initialized!"); + } + + @Override + public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { + BlockingQueue> queue = pendingSessionMap.remove(msgUid); + if (queue != null) { + try { + queue.put(inputStream); + } catch (InterruptedException e) { + log.warn("Failed to report created session!"); + Thread.currentThread().interrupt(); + } + } else { + log.warn("Failed to lookup pending session!"); + } + } + + @Override + public StreamObserver handleMsgs( + StreamObserver responseObserver) { + log.info("Processing new session."); + return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); + } + + + @PreDestroy + public void stop() { + if (server != null) { + log.info("Going to onStop RPC server"); + server.shutdownNow(); + try { + server.awaitTermination(); + log.info("RPC server stopped!"); + } catch (InterruptedException e) { + log.warn("Failed to onStop RPC server!"); + Thread.currentThread().interrupt(); + } + } + } + + + @Override + public void broadcast(RpcBroadcastMsg msg) { + listener.onBroadcastMsg(msg); + } + + private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { + BlockingQueue> queue = new ArrayBlockingQueue<>(1); + pendingSessionMap.put(msg.getMsgUid(), queue); + listener.onRpcSessionCreateRequestMsg(msg); + try { + StreamObserver observer = queue.take(); + log.info("Processed new session."); + return observer; + } catch (Exception e) { + log.info("Failed to process session.", e); + throw new RuntimeException(e); + } + } + + @Override + public void tell(ClusterAPIProtos.ClusterMessage message) { + listener.onSendMsg(message); + } + + @Override + public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { + listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); + } + + @Override + public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { + ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage + .newBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(serverAddress.getHost()) + .setPort(serverAddress.getPort()) + .build()) + .setMessageType(msgType) + .setPayload(ByteString.copyFrom(data)).build(); + listener.onSendMsg(msg); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java new file mode 100644 index 0000000000..637924fca9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import io.grpc.stub.StreamObserver; +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +public interface ClusterRpcService { + + void init(RpcMsgListener listener); + + void broadcast(RpcBroadcastMsg msg); + + void onSessionCreated(UUID msgUid, StreamObserver inputStream); + + void tell(ClusterAPIProtos.ClusterMessage message); + + void tell(ServerAddress serverAddress, TbActorMsg actorMsg); + + void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java new file mode 100644 index 0000000000..aff0bc4a41 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +import java.io.Closeable; +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +@Slf4j +public final class GrpcSession implements Closeable { + private final UUID sessionId; + private final boolean client; + private final GrpcSessionListener listener; + private final ManagedChannel channel; + private StreamObserver inputStream; + private StreamObserver outputStream; + + private boolean connected; + private ServerAddress remoteServer; + + public GrpcSession(GrpcSessionListener listener) { + this(null, listener, null); + } + + public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { + this.sessionId = UUID.randomUUID(); + this.listener = listener; + if (remoteServer != null) { + this.client = true; + this.connected = true; + this.remoteServer = remoteServer; + } else { + this.client = false; + } + this.channel = channel; + } + + public void initInputStream() { + this.inputStream = new StreamObserver() { + @Override + public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { + if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { + connected = true; + ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); + remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); + listener.onConnected(GrpcSession.this); + } + if (connected) { + listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); + } + } + + @Override + public void onError(Throwable t) { + listener.onError(GrpcSession.this, t); + } + + @Override + public void onCompleted() { + outputStream.onCompleted(); + listener.onDisconnected(GrpcSession.this); + } + }; + } + + public void initOutputStream() { + if (client) { + listener.onConnected(GrpcSession.this); + } + } + + public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { + if (connected) { + try { + outputStream.onNext(msg); + } catch (Throwable t) { + try { + outputStream.onError(t); + } catch (Throwable t2) { + } + listener.onError(GrpcSession.this, t); + } + } else { + log.warn("[{}] Failed to send message due to closed session!", sessionId); + } + } + + @Override + public void close() { + connected = false; + try { + outputStream.onCompleted(); + } catch (IllegalStateException e) { + log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); + } + if (channel != null) { + channel.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java similarity index 59% rename from application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java index 3d4f23a796..a6ecf967d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.rpc; +package org.thingsboard.server.service.cluster.rpc; -import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; /** - * Created by ashvayka on 16.04.18. + * @author Andrew Shvayka */ -public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { +public interface GrpcSessionListener { - /** - * Handles the RPC response from the Device Actor (Transport). - * - * @param response the RPC response - */ - void processRpcResponseFromDevice(FromDeviceRpcResponse response); + void onConnected(GrpcSession session); + void onDisconnected(GrpcSession session); + + void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); + + void onError(GrpcSession session, Throwable t); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java new file mode 100644 index 0000000000..2ff37b39e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +/** + * @author Andrew Shvayka + */ + +public interface RpcMsgListener { + void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); + void onSendMsg(ClusterAPIProtos.ClusterMessage msg); + void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); + void onBroadcastMsg(RpcBroadcastMsg msg); +} diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index 4a781b8673..a5d3ab465e 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -25,5 +27,8 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); + ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, + TbActorMsg msg); + } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 8d89059488..45bb9f78fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,19 +15,25 @@ */ package org.thingsboard.server.service.encoding; +import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; +import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; + + @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); @Override public Optional decode(byte[] byteArray) { try { @@ -36,7 +42,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -45,4 +51,18 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } + @Override + public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, + TbActorMsg msg) { + return ClusterAPIProtos.ClusterMessage + .newBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(serverAddress.getHost()) + .setPort(serverAddress.getPort()) + .build()) + .setMessageType(CLUSTER_ACTOR_MESSAGE) + .setPayload(ByteString.copyFrom(encode(msg))).build(); + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java b/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java rename to application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java index 267b34b204..a6a881ecde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java +++ b/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.environment; +package org.thingsboard.server.service.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("ThingsBoard server environment: ", log); + Environment.logEnv("Thingsboard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java rename to application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java index 50da4f4a1e..f4b14144fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java @@ -13,20 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.service.executors; -import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.AbstractListeningExecutor; -@Data @Component -public class TbQueueTransportNotificationSettings { +public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { - @Value("${queue.transport.notifications_topic}") - private String notificationsTopic; + @Value("${actors.cluster.grpc_callback_thread_pool_size}") + private int grpcCallbackExecutorThreadPoolSize; - @Value("${queue.transport.poll_interval}") - private long transportPollInterval; + @Override + protected int getThreadPollSize() { + return grpcCallbackExecutorThreadPoolSize; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 87164737c7..ef03e3ec43 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,15 +221,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } - try { - conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); - } catch (Exception e) { - } - try { - long ts = System.currentTimeMillis(); - conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) { - } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index e7f21a18d0..08a5fdd3e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -203,8 +203,7 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("entity_type"), stringColumn("event_type"), stringColumn("event_uid"), - stringColumn("body"), - new CassandraToSqlEventTsColumn()), + stringColumn("body")), new CassandraToSqlTable("relation", idColumn("from_id"), stringColumn("from_type"), @@ -246,9 +245,7 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("zip"), stringColumn("phone"), stringColumn("email"), - stringColumn("additional_info"), - booleanColumn("isolated_tb_core"), - booleanColumn("isolated_tb_rule_engine")), + stringColumn("additional_info")), new CassandraToSqlTable("user_credentials", idColumn("id"), idColumn("user_id"), diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java deleted file mode 100644 index 8e369c66c6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.install.migrate; - -import com.datastax.driver.core.Row; - -import java.util.UUID; - -import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; - -public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { - - CassandraToSqlEventTsColumn() { - super("id", "ts", CassandraToSqlColumnType.BIGINT, null); - } - - @Override - public String getColumnValue(Row row) { - UUID id = row.getUUID(getIndex()); - long ts = getTs(id); - return ts + ""; - } - - private long getTs(UUID uuid) { - return (uuid.timestamp() - EPOCH_DIFF) / 10000; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java deleted file mode 100644 index 926b08f38b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import com.google.protobuf.ByteString; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; - -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -@Service -@Slf4j -public class DefaultTbClusterService implements TbClusterService { - - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - private final AtomicInteger toCoreMsgs = new AtomicInteger(0); - private final AtomicInteger toCoreNfs = new AtomicInteger(0); - private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); - private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); - private final AtomicInteger toTransportNfs = new AtomicInteger(0); - - private final TbQueueProducerProvider producerProvider; - private final PartitionService partitionService; - private final DataDecodingEncodingService encodingService; - - public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { - this.producerProvider = producerProvider; - this.partitionService = partitionService; - this.encodingService = encodingService; - } - - @Override - public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); - log.trace("PUSHING msg: {} to:{}", msg, tpi); - byte[] msgBytes = encodingService.encode(msg); - ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() - .setRequestIdMSB(response.getId().getMostSignificantBits()) - .setRequestIdLSB(response.getId().getLeastSignificantBits()) - .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); - response.getResponse().ifPresent(builder::setResponse); - ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); - toCoreNfs.incrementAndGet(); - } - - @Override - public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { - log.trace("PUSHING msg: {} to:{}", msg, tpi); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toRuleEngineMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { - if (tenantId.isNullUid()) { - if (entityId.getEntityType().equals(EntityType.TENANT)) { - tenantId = new TenantId(entityId.getId()); - } else { - log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); - return; - } - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); - toRuleEngineMsgs.incrementAndGet(); - } - - @Override - public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() - .setRequestIdMSB(response.getId().getMostSignificantBits()) - .setRequestIdLSB(response.getId().getLeastSignificantBits()) - .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); - response.getResponse().ifPresent(builder::setResponse); - ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); - toRuleEngineNfs.incrementAndGet(); - } - - @Override - public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); - toTransportNfs.incrementAndGet(); - } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - private void broadcast(ComponentLifecycleMsg msg) { - byte[] msgBytes = encodingService.encode(msg); - TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); - if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { - TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); - Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); - for (String serviceId : tbCoreServices) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); - toCoreNfs.incrementAndGet(); - } - // No need to push notifications twice - tbRuleEngineServices.removeAll(tbCoreServices); - } - for (String serviceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } - } - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int toCoreMsgCnt = toCoreMsgs.getAndSet(0); - int toCoreNfsCnt = toCoreNfs.getAndSet(0); - int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); - int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); - int toTransportNfsCnt = toTransportNfs.getAndSet(0); - if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { - log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", - toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java deleted file mode 100644 index a5abd77ad5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; -import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@Slf4j -public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { - - @Value("${queue.core.poll-interval}") - private long pollDuration; - @Value("${queue.core.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.core.stats.enabled:false}") - private boolean statsEnabled; - - private final TbQueueConsumer> mainConsumer; - private final DeviceStateService stateService; - private final TbLocalSubscriptionService localSubscriptionService; - private final SubscriptionManagerService subscriptionManagerService; - private final TbCoreDeviceRpcService tbCoreDeviceRpcService; - private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - - public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, - DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, - SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, - TbCoreDeviceRpcService tbCoreDeviceRpcService) { - super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); - this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); - this.stateService = stateService; - this.localSubscriptionService = localSubscriptionService; - this.subscriptionManagerService = subscriptionManagerService; - this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; - } - - @PostConstruct - public void init() { - super.init("tb-core-consumer", "tb-core-notifications-consumer"); - } - - @PreDestroy - public void destroy(){ - super.destroy(); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceType().equals(getServiceType())) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @Override - protected void launchMainConsumers() { - consumersExecutor.submit(() -> { - while (!stopped) { - try { - List> msgs = mainConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - TbPackProcessingContext> ctx = new TbPackProcessingContext<>( - processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - pendingMap.forEach((id, msg) -> { - log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, ctx); - try { - ToCoreMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToSubscriptionMgrMsg()) { - log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); - forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); - } else if (toCoreMsg.hasToDeviceActorMsg()) { - log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); - } else if (toCoreMsg.hasDeviceStateServiceMsg()) { - log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); - forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); - } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process message: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); - } - mainConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - log.info("TB Core Consumer stopped."); - }); - } - - @Override - protected ServiceType getServiceType() { - return ServiceType.TB_CORE; - } - - @Override - protected long getNotificationPollDuration() { - return pollDuration; - } - - @Override - protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; - } - - @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { - ToCoreNotificationMsg toCoreNotification = msg.getValue(); - if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreNotification.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); - } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - if (statsEnabled) { - stats.log(toCoreNotification); - } - } - - private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) - , proto.getResponse(), error); - tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); - callback.onSuccess(); - } - - @Scheduled(fixedDelayString = "${queue.core.stats.print-interval-ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbCallback callback) { - if (msg.hasSubUpdate()) { - localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); - } else { - throwNotHandled(msg, callback); - } - } - - private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { - if (msg.hasAttributeSub()) { - subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); - } else if (msg.hasTelemetrySub()) { - subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); - } else if (msg.hasSubClose()) { - TbSubscriptionCloseProto closeProto = msg.getSubClose(); - subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); - } else if (msg.hasTsUpdate()) { - TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); - subscriptionManagerService.onTimeSeriesUpdate( - new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), - TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); - } else if (msg.hasAttrUpdate()) { - TbAttributeUpdateProto proto = msg.getAttrUpdate(); - subscriptionManagerService.onAttributesUpdate( - new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), - proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); - } else { - throwNotHandled(msg, callback); - } - if (statsEnabled) { - stats.log(msg); - } - } - - private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbCallback callback) { - if (statsEnabled) { - stats.log(deviceStateServiceMsg); - } - stateService.onQueueMsg(deviceStateServiceMsg, callback); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); - } - - private void throwNotHandled(Object msg, TbCallback callback) { - log.warn("Message not handled: {}", msg); - callback.onFailure(new RuntimeException("Message not handled!")); - } - - @Override - protected void stopMainConsumers() { - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java deleted file mode 100644 index 0c48c54412..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import akka.actor.ActorRef; -import com.google.protobuf.ProtocolStringList; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceQueue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; -import org.thingsboard.server.service.stats.RuleEngineStatisticsService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -@Service -@TbRuleEngineComponent -@Slf4j -public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - - @Value("${queue.rule-engine.poll-interval}") - private long pollDuration; - @Value("${queue.rule-engine.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:true}") - private boolean statsEnabled; - - private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; - private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; - private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final RuleEngineStatisticsService statisticsService; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); - private ExecutorService submitExecutor; - - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, - TbRuleEngineSubmitStrategyFactory submitStrategyFactory, - TbQueueRuleEngineSettings ruleEngineSettings, - TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, - ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { - super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); - this.statisticsService = statisticsService; - this.ruleEngineSettings = ruleEngineSettings; - this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.submitStrategyFactory = submitStrategyFactory; - this.processingStrategyFactory = processingStrategyFactory; - } - - @PostConstruct - public void init() { - super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); - for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { - consumerConfigurations.putIfAbsent(configuration.getName(), configuration); - consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); - consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); - } - submitExecutor = Executors.newSingleThreadExecutor(); - } - - @PreDestroy - public void stop() { - super.destroy(); - if (submitExecutor != null) { - submitExecutor.shutdownNow(); - } - ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceType().equals(getServiceType())) { - ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); - consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); - } - } - - @Override - protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); - } - - @Override - protected void stopMainConsumers() { - consumers.values().forEach(TbQueueConsumer::unsubscribe); - } - - private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { - consumersExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = consumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); - TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); - - submitStrategy.init(msgs); - - while (!stopped) { - TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); - submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { - log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); - try { - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); - } else { - callback.onSuccess(); - } - } catch (Exception e) { - callback.onFailure(new RuleEngineException(e.getMessage())); - } - })); - - boolean timeout = false; - if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { - timeout = true; - } - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (statsEnabled) { - stats.log(result, decision.isCommit()); - } - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - log.info("TB Rule Engine Consumer stopped."); - }); - } - - @Override - protected ServiceType getServiceType() { - return ServiceType.TB_RULE_ENGINE; - } - - @Override - protected long getNotificationPollDuration() { - return pollDuration; - } - - @Override - protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; - } - - @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { - ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } else { - callback.onSuccess(); - } - } - - private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); - QueueToRuleEngineMsg msg; - ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); - Set relationTypes = null; - if (relationTypesList != null) { - if (relationTypesList.size() == 1) { - relationTypes = Collections.singleton(relationTypesList.get(0)); - } else { - relationTypes = new HashSet<>(relationTypesList); - } - } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.tell(msg, ActorRef.noSender()); - } - - @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") - public void printStats() { - if (statsEnabled) { - long ts = System.currentTimeMillis(); - consumerStats.forEach((queue, stats) -> { - stats.printStats(); - statisticsService.reportQueueStats(ts, stats); - }); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java deleted file mode 100644 index aae3cef0ac..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") -public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - - private final TenantService tenantService; - - public DefaultTenantRoutingInfoService(TenantService tenantService) { - this.tenantService = tenantService; - } - - @Override - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - Tenant tenant = tenantService.findTenantById(tenantId); - if (tenant != null) { - return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); - } else { - throw new RuntimeException("Tenant not found!"); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java deleted file mode 100644 index cc722720d5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; - -import java.util.UUID; - -public interface TbClusterService { - - void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback); - - void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback); - - void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); - - void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); - - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); - - void pushNotificationToRuleEngine(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - - void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); - - void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java deleted file mode 100644 index 2dcdb0e0c2..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; - -public interface TbCoreConsumerService extends ApplicationListener { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java deleted file mode 100644 index f093cc885a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; - -import java.util.UUID; - -@Slf4j -public class TbMsgPackCallback implements TbMsgCallback { - private final UUID id; - private final TenantId tenantId; - private final TbMsgPackProcessingContext ctx; - - public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { - this.id = id; - this.tenantId = tenantId; - this.ctx = ctx; - } - - @Override - public void onSuccess() { - log.trace("[{}] ON SUCCESS", id); - ctx.onSuccess(id); - } - - @Override - public void onFailure(RuleEngineException e) { - log.trace("[{}] ON FAILURE", id, e); - ctx.onFailure(tenantId, id, e); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java deleted file mode 100644 index 7e78ac6f5f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Getter; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -public class TbMsgPackProcessingContext { - - private final TbRuleEngineSubmitStrategy submitStrategy; - - private final AtomicInteger pendingCount; - private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - @Getter - private final ConcurrentMap> pendingMap; - @Getter - private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); - @Getter - private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - @Getter - private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - - public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { - this.submitStrategy = submitStrategy; - this.pendingMap = submitStrategy.getPendingMap(); - this.pendingCount = new AtomicInteger(pendingMap.size()); - } - - public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { - return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); - } - - public void onSuccess(UUID id) { - TbProtoQueueMsg msg; - boolean empty = false; - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - successMap.put(id, msg); - submitStrategy.onSuccess(id); - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } - - public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { - TbProtoQueueMsg msg; - boolean empty = false; - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - failedMap.put(id, msg); - exceptionsMap.putIfAbsent(tenantId, e); - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java deleted file mode 100644 index e9f1224625..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; - -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -public class TbPackProcessingContext { - - private final AtomicInteger pendingCount; - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap failedMap; - - public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap failedMap) { - this.processingTimeoutLatch = processingTimeoutLatch; - this.pendingCount = new AtomicInteger(ackMap.size()); - this.ackMap = ackMap; - this.failedMap = failedMap; - } - - public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { - return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); - } - - public void onSuccess(UUID id) { - boolean empty = false; - T msg = ackMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - } - if (empty) { - processingTimeoutLatch.countDown(); - } else { - if (log.isTraceEnabled()) { - log.trace("Items left: {}", ackMap.size()); - for (T t : ackMap.values()) { - log.trace("left item: {}", t); - } - } - } - } - - public void onFailure(UUID id, Throwable t) { - boolean empty = false; - T msg = ackMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - failedMap.put(id, msg); - if (log.isTraceEnabled()) { - log.trace("Items left: {}", ackMap.size()); - for (T v : ackMap.values()) { - log.trace("left item: {}", v); - } - } - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } - - public ConcurrentMap getAckMap() { - return ackMap; - } - - public ConcurrentMap getFailedMap() { - return failedMap; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java deleted file mode 100644 index 19966b4566..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; - -public interface TbRuleEngineConsumerService extends ApplicationListener { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java deleted file mode 100644 index 40017d2b40..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -@Data -public class TbRuleEngineConsumerStats { - - public static final String TOTAL_MSGS = "totalMsgs"; - public static final String SUCCESSFUL_MSGS = "successfulMsgs"; - public static final String TMP_TIMEOUT = "tmpTimeout"; - public static final String TMP_FAILED = "tmpFailed"; - public static final String TIMEOUT_MSGS = "timeoutMsgs"; - public static final String FAILED_MSGS = "failedMsgs"; - public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; - public static final String FAILED_ITERATIONS = "failedIterations"; - - private final AtomicInteger totalMsgCounter = new AtomicInteger(0); - private final AtomicInteger successMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger failedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger successIterationsCounter = new AtomicInteger(0); - private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); - - private final Map counters = new HashMap<>(); - private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); - private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); - - private final String queueName; - - public TbRuleEngineConsumerStats(String queueName) { - this.queueName = queueName; - counters.put(TOTAL_MSGS, totalMsgCounter); - counters.put(SUCCESSFUL_MSGS, successMsgCounter); - counters.put(TIMEOUT_MSGS, timeoutMsgCounter); - counters.put(FAILED_MSGS, failedMsgCounter); - - counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); - counters.put(TMP_FAILED, tmpFailedMsgCounter); - counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); - counters.put(FAILED_ITERATIONS, failedIterationsCounter); - } - - public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { - int success = msg.getSuccessMap().size(); - int pending = msg.getPendingMap().size(); - int failed = msg.getFailedMap().size(); - totalMsgCounter.addAndGet(success + pending + failed); - successMsgCounter.addAndGet(success); - msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); - if (finalIterationForPack) { - if (pending > 0 || failed > 0) { - timeoutMsgCounter.addAndGet(pending); - failedMsgCounter.addAndGet(failed); - if (pending > 0) { - msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); - } - if (failed > 0) { - msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); - } - failedIterationsCounter.incrementAndGet(); - } else { - successIterationsCounter.incrementAndGet(); - } - } else { - failedIterationsCounter.incrementAndGet(); - tmpTimeoutMsgCounter.addAndGet(pending); - tmpFailedMsgCounter.addAndGet(failed); - if (pending > 0) { - msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); - } - if (failed > 0) { - msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); - } - } - msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); - } - - private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { - ToRuleEngineMsg reMsg = m.getValue(); - return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); - } - - public void printStats() { - int total = totalMsgCounter.get(); - if (total > 0) { - StringBuilder stats = new StringBuilder(); - counters.forEach((label, value) -> { - stats.append(label).append(" = [").append(value.get()).append("] "); - }); - log.info("[{}] Stats: {}", queueName, stats); - } - } - - public void reset() { - counters.values().forEach(counter -> counter.set(0)); - tenantStats.clear(); - tenantExceptions.clear(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java deleted file mode 100644 index fb36f5064b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -@Data -public class TbTenantRuleEngineStats { - - private final UUID tenantId; - - private final AtomicInteger totalMsgCounter = new AtomicInteger(0); - private final AtomicInteger successMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger failedMsgCounter = new AtomicInteger(0); - - private final Map counters = new HashMap<>(); - - public TbTenantRuleEngineStats(UUID tenantId) { - this.tenantId = tenantId; - counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); - counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); - counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); - counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); - - counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); - counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); - } - - public void logSuccess() { - totalMsgCounter.incrementAndGet(); - successMsgCounter.incrementAndGet(); - } - - public void logFailed() { - totalMsgCounter.incrementAndGet(); - failedMsgCounter.incrementAndGet(); - } - - public void logTimeout() { - totalMsgCounter.incrementAndGet(); - timeoutMsgCounter.incrementAndGet(); - } - - public void logTmpFailed() { - totalMsgCounter.incrementAndGet(); - tmpFailedMsgCounter.incrementAndGet(); - } - - public void logTmpTimeout() { - totalMsgCounter.incrementAndGet(); - tmpTimeoutMsgCounter.incrementAndGet(); - } - - public void printStats() { - int total = totalMsgCounter.get(); - if (total > 0) { - StringBuilder stats = new StringBuilder(); - counters.forEach((label, value) -> { - stats.append(label).append(" = [").append(value.get()).append("]"); - }); - log.info("[{}] Stats: {}", tenantId, stats); - } - } - - public void reset() { - counters.values().forEach(counter -> counter.set(0)); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java deleted file mode 100644 index c2705fcbdc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.EventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.TbPackCallback; -import org.thingsboard.server.service.queue.TbPackProcessingContext; - -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public abstract class AbstractConsumerService implements ApplicationListener { - - protected volatile ExecutorService consumersExecutor; - protected volatile ExecutorService notificationsConsumerExecutor; - protected volatile boolean stopped = false; - - protected final ActorSystemContext actorContext; - protected final DataDecodingEncodingService encodingService; - - protected final TbQueueConsumer> nfConsumer; - - public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { - this.actorContext = actorContext; - this.encodingService = encodingService; - this.nfConsumer = nfConsumer; - } - - public void init(String mainConsumerThreadName, String nfConsumerThreadName) { - this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumers(); - } - - protected abstract ServiceType getServiceType(); - - protected abstract void launchMainConsumers(); - - protected abstract void stopMainConsumers(); - - protected abstract long getNotificationPollDuration(); - - protected abstract long getNotificationPackProcessingTimeout(); - - protected void launchNotificationsConsumer() { - notificationsConsumerExecutor.submit(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(getNotificationPollDuration()); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - TbPackProcessingContext> ctx = new TbPackProcessingContext<>( - processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - pendingMap.forEach((id, msg) -> { - log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, ctx); - try { - handleNotification(id, msg, callback); - } catch (Throwable e) { - log.warn("[{}] Failed to process notification: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { - ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain notifications from queue.", e); - try { - Thread.sleep(getNotificationPollDuration()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new notifications", e2); - } - } - } - } - log.info("TB Notifications Consumer stopped."); - }); - } - - protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; - - @PreDestroy - public void destroy() { - stopped = true; - - stopMainConsumers(); - - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - - if (consumersExecutor != null) { - consumersExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java deleted file mode 100644 index bef733ec22..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Collectors; - -public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { - - protected final String queueName; - protected List orderedMsgList; - private volatile boolean stopped; - - public AbstractTbRuleEngineSubmitStrategy(String queueName) { - this.queueName = queueName; - } - - protected abstract void doOnSuccess(UUID id); - - @Override - public void init(List> msgs) { - orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); - } - - @Override - public ConcurrentMap> getPendingMap() { - return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); - for (IdMsgPair pair : orderedMsgList) { - if (reprocessMap.containsKey(pair.uuid)) { - newOrderedMsgList.add(pair); - } - } - orderedMsgList = newOrderedMsgList; - } - - @Override - public void onSuccess(UUID id) { - if (!stopped) { - doOnSuccess(id); - } - } - - @Override - public void stop() { - stopped = true; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java deleted file mode 100644 index d0b1f7f99a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -@Slf4j -public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private final int batchSize; - private final AtomicInteger packIdx = new AtomicInteger(0); - private final Map> pendingPack = new LinkedHashMap<>(); - private volatile BiConsumer> msgConsumer; - - public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { - super(queueName); - this.batchSize = batchSize; - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - submitNext(); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - packIdx.set(0); - } - - @Override - protected void doOnSuccess(UUID id) { - boolean endOfPendingPack; - synchronized (pendingPack) { - TbProtoQueueMsg msg = pendingPack.remove(id); - endOfPendingPack = msg != null && pendingPack.isEmpty(); - } - if (endOfPendingPack) { - packIdx.incrementAndGet(); - submitNext(); - } - } - - private void submitNext() { - int listSize = orderedMsgList.size(); - int startIdx = Math.min(packIdx.get() * batchSize, listSize); - int endIdx = Math.min(startIdx + batchSize, listSize); - synchronized (pendingPack) { - pendingPack.clear(); - for (int i = startIdx; i < endIdx; i++) { - IdMsgPair pair = orderedMsgList.get(i); - pendingPack.put(pair.uuid, pair.msg); - } - } - int submitSize = pendingPack.size(); - if (log.isInfoEnabled() && submitSize > 0) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); - } - pendingPack.forEach(msgConsumer); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ffd1dd49d1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - public BurstTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); - } - orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); - } - - @Override - protected void doOnSuccess(UUID id) { - - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ae5993cb1c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Queue; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; - -@Slf4j -public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private volatile BiConsumer> msgConsumer; - private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); - private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); - - public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void init(List> msgs) { - super.init(msgs); - initMaps(); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - entityIdToListMap.forEach((entityId, queue) -> { - IdMsgPair msg = queue.peek(); - if (msg != null) { - msgConsumer.accept(msg.uuid, msg.msg); - } - }); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - initMaps(); - } - - @Override - protected void doOnSuccess(UUID id) { - EntityId entityId = msgToEntityIdMap.get(id); - if (entityId != null) { - Queue queue = entityIdToListMap.get(entityId); - if (queue != null) { - IdMsgPair next = null; - synchronized (queue) { - IdMsgPair expected = queue.peek(); - if (expected != null && expected.uuid.equals(id)) { - queue.poll(); - next = queue.peek(); - } - } - if (next != null) { - msgConsumer.accept(next.uuid, next.msg); - } - } - } - } - - private void initMaps() { - msgToEntityIdMap.clear(); - entityIdToListMap.clear(); - for (IdMsgPair pair : orderedMsgList) { - EntityId entityId = getEntityId(pair.msg.getValue()); - if (entityId != null) { - msgToEntityIdMap.put(pair.uuid, entityId); - entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); - } - } - } - - protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java deleted file mode 100644 index cd8a97e82c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.UUID; - -@Slf4j -public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { - - public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { - try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); - return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - } catch (InvalidProtocolBufferException e) { - log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); - return null; - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ef45b983fc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -@Slf4j -public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private final AtomicInteger msgIdx = new AtomicInteger(0); - private volatile BiConsumer> msgConsumer; - private volatile UUID expectedMsgId; - - public SequentialTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - msgIdx.set(0); - submitNext(); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - } - - @Override - protected void doOnSuccess(UUID id) { - if (expectedMsgId.equals(id)) { - msgIdx.incrementAndGet(); - submitNext(); - } - } - - private void submitNext() { - int listSize = orderedMsgList.size(); - int idx = msgIdx.get(); - if (idx < listSize) { - IdMsgPair pair = orderedMsgList.get(idx); - expectedMsgId = pair.uuid; - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); - } - msgConsumer.accept(pair.uuid, pair.msg); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java deleted file mode 100644 index e818ffb9d7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.Getter; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; - -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; - -public class TbRuleEngineProcessingResult { - - @Getter - private final boolean success; - @Getter - private final boolean timeout; - @Getter - private final TbMsgPackProcessingContext ctx; - - public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { - this.timeout = timeout; - this.ctx = ctx; - this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); - } - - public ConcurrentMap> getPendingMap() { - return ctx.getPendingMap(); - } - - public ConcurrentMap> getSuccessMap() { - return ctx.getSuccessMap(); - } - - public ConcurrentMap> getFailedMap() { - return ctx.getFailedMap(); - } - - public ConcurrentMap getExceptionsMap() { - return ctx.getExceptionsMap(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java deleted file mode 100644 index cfba85f5dc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -public interface TbRuleEngineProcessingStrategy { - - TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java deleted file mode 100644 index bbf283e962..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -@Component -@Slf4j -public class TbRuleEngineProcessingStrategyFactory { - - public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { - switch (configuration.getType()) { - case "SKIP_ALL_FAILURES": - return new SkipStrategy(name); - case "RETRY_ALL": - return new RetryStrategy(name, true, true, true, configuration); - case "RETRY_FAILED": - return new RetryStrategy(name, false, true, false, configuration); - case "RETRY_TIMED_OUT": - return new RetryStrategy(name, false, false, true, configuration); - case "RETRY_FAILED_AND_TIMED_OUT": - return new RetryStrategy(name, false, true, true, configuration); - default: - throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); - } - } - - private static class RetryStrategy implements TbRuleEngineProcessingStrategy { - private final String queueName; - private final boolean retrySuccessful; - private final boolean retryFailed; - private final boolean retryTimeout; - private final int maxRetries; - private final double maxAllowedFailurePercentage; - private final long pauseBetweenRetries; - - private int initialTotalCount; - private int retryCount; - - public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { - this.queueName = queueName; - this.retrySuccessful = retrySuccessful; - this.retryFailed = retryFailed; - this.retryTimeout = retryTimeout; - this.maxRetries = configuration.getRetries(); - this.maxAllowedFailurePercentage = configuration.getFailurePercentage(); - this.pauseBetweenRetries = configuration.getPauseBetweenRetries(); - } - - @Override - public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - if (result.isSuccess()) { - return new TbRuleEngineProcessingDecision(true, null); - } else { - if (retryCount == 0) { - initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); - } - retryCount++; - double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); - if (maxRetries > 0 && retryCount > maxRetries) { - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); - return new TbRuleEngineProcessingDecision(true, null); - } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); - return new TbRuleEngineProcessingDecision(true, null); - } else { - ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); - if (retryFailed) { - result.getFailedMap().forEach(toReprocess::put); - } - if (retryTimeout) { - result.getPendingMap().forEach(toReprocess::put); - } - if (retrySuccessful) { - result.getSuccessMap().forEach(toReprocess::put); - } - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); - if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - if (pauseBetweenRetries > 0) { - try { - Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - return new TbRuleEngineProcessingDecision(false, toReprocess); - } - } - } - } - - private static class SkipStrategy implements TbRuleEngineProcessingStrategy { - - private final String queueName; - - public SkipStrategy(String name) { - this.queueName = name; - } - - @Override - public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - if (!result.isSuccess()) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); - } - if (log.isTraceEnabled()) { - result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - if (log.isTraceEnabled()) { - result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - return new TbRuleEngineProcessingDecision(true, null); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java deleted file mode 100644 index 7b22da97db..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.function.BiConsumer; - -public interface TbRuleEngineSubmitStrategy { - - void init(List> msgs); - - ConcurrentMap> getPendingMap(); - - void submitAttempt(BiConsumer> msgConsumer); - - void update(ConcurrentMap> reprocessMap); - - void onSuccess(UUID id); - - void stop(); -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java deleted file mode 100644 index 90eb8561a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; - -@Component -@Slf4j -public class TbRuleEngineSubmitStrategyFactory { - - public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { - switch (configuration.getType()) { - case "BURST": - return new BurstTbRuleEngineSubmitStrategy(name); - case "BATCH": - return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); - case "SEQUENTIAL_BY_ORIGINATOR": - return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL_BY_TENANT": - return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL": - return new SequentialTbRuleEngineSubmitStrategy(name); - default: - throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java new file mode 100644 index 0000000000..5d303b54ab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -0,0 +1,222 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +public class DefaultDeviceRpcService implements DeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService rpcService; + + @Autowired + private DeviceService deviceService; + + @Autowired + @Lazy + private ActorService actorService; + + private ScheduledExecutorService rpcCallBackExecutor; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + @PostConstruct + public void initExecutor() { + rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); + } + + @PreDestroy + public void shutdownExecutor() { + if (rpcCallBackExecutor != null) { + rpcCallBackExecutor.shutdownNow(); + } + } + + @Override + public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); + } + + @Override + public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); + if (routingService.getCurrentServer().equals(requestOriginAddress)) { + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } else { + ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); + builder.setRequestIdMSB(response.getId().getMostSignificantBits()); + builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); + response.getResponse().ifPresent(builder::setResponse); + if (response.getError().isPresent()) { + builder.setError(response.getError().get().ordinal()); + } else { + builder.setError(-1); + } + rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); + } + } + + @Override + public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId, localToDeviceRpcRequests); + } + + @Override + public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = localToDeviceRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.FromDeviceRPCResponseProto proto; + try { + proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); + processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); + } + + @Override + public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { + ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); + forward(deviceId, rpcMsg); + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originHost", routingService.getCurrentServer().getHost()); + metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + forward(msg.getDeviceId(), rpcMsg); + } + + private void forward(DeviceId deviceId, T msg) { + actorService.onMsg(new SendToClusterMsg(deviceId, msg)); + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + rpcCallBackExecutor.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = requestsMap.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java deleted file mode 100644 index f6e5eb8b43..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ /dev/null @@ -1,199 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import akka.actor.ActorRef; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -@TbCoreComponent -public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - private final DeviceService deviceService; - private final TbClusterService clusterService; - private final TbServiceInfoProvider serviceInfoProvider; - private final ActorSystemContext actorContext; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - private Optional tbRuleEngineRpcService; - private ScheduledExecutorService scheduler; - private String serviceId; - - public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, - ActorSystemContext actorContext) { - this.deviceService = deviceService; - this.clusterService = clusterService; - this.serviceInfoProvider = serviceInfoProvider; - this.actorContext = actorContext; - } - - @Autowired(required = false) - public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { - this.tbRuleEngineRpcService = tbRuleEngineRpcService; - } - - @PostConstruct - public void initExecutor() { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); - serviceId = serviceInfoProvider.getServiceId(); - } - - @PreDestroy - public void shutdownExecutor() { - if (scheduler != null) { - scheduler.shutdownNow(); - } - } - - @Override - public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleToRuleEngineTimeout(request, requestId); - } - - @Override - public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { - ToDeviceRpcRequest request = rpcMsg.getMsg(); - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, rpcMsg); - actorContext.tell(rpcMsg, ActorRef.noSender()); - scheduleToDeviceTimeout(request, requestId); - } - - @Override - public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); - if (request != null) { - sendRpcResponseToTbRuleEngine(request.getServiceId(), response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { - if (serviceId.equals(originServiceId)) { - if (tbRuleEngineRpcService.isPresent()) { - tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - clusterService.pushNotificationToRuleEngine(originServiceId, response, null); - } - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originServiceId", serviceId); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to rule engine request.", requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout for processing to rule engine request.", requestId); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to device request.", requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout for to device request.", requestId); - localToDeviceRpcRequests.remove(requestId); - }, timeout, TimeUnit.MILLISECONDS); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java deleted file mode 100644 index 0ec730b7dc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.queue.TbClusterService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -@Service -@TbRuleEngineComponent -@Slf4j -public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { - - private final PartitionService partitionService; - private final TbClusterService clusterService; - private final TbServiceInfoProvider serviceInfoProvider; - - - private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); - - private Optional tbCoreRpcService; - private ScheduledExecutorService scheduler; - private String serviceId; - - public DefaultTbRuleEngineRpcService(PartitionService partitionService, - TbClusterService clusterService, - TbServiceInfoProvider serviceInfoProvider) { - this.partitionService = partitionService; - this.clusterService = clusterService; - this.serviceInfoProvider = serviceInfoProvider; - } - - @Autowired(required = false) - public void setTbCoreRpcService(Optional tbCoreRpcService) { - this.tbCoreRpcService = tbCoreRpcService; - } - - @PostConstruct - public void initExecutor() { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); - serviceId = serviceInfoProvider.getServiceId(); - } - - @PreDestroy - public void shutdownExecutor() { - if (scheduler != null) { - scheduler.shutdownNow(); - } - } - - @Override - public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { - TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId) - .setPayload(body).build(); - TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() - .setSessionIdMSB(sessionId.getMostSignificantBits()) - .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setToServerResponse(responseMsg) - .build(); - clusterService.pushNotificationToTransport(serviceId, msg, null); - } - - @Override - public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - forwardRpcRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - sendRpcResponseToTbCore(src.getOriginServiceId(), response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - - @Override - public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = toDeviceRpcRequests.remove(requestId); - if (consumer != null) { - scheduler.submit(() -> consumer.accept(response)); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - toDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId); - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); - if (tpi.isMyPartition()) { - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - if (tbCoreRpcService.isPresent()) { - tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); - clusterService.pushMsgToCore(rpcMsg, null); - } - } - - private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { - if (serviceId.equals(originServiceId)) { - if (tbCoreRpcService.isPresent()) { - tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - clusterService.pushNotificationToCore(originServiceId, response, null); - } - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = toDeviceRpcRequests.remove(requestId); - if (consumer != null) { - scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); - } - }, timeout, TimeUnit.MILLISECONDS); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java new file mode 100644 index 0000000000..feb5d58c3d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Created by ashvayka on 16.04.18. + */ +public interface DeviceRpcService { + + void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); + + void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); + + void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); + + void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); + + void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); + + void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index c1e5e2e038..78ea0c5660 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,6 +19,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java deleted file mode 100644 index 896745c48e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Handles REST API calls that contain RPC requests to Device. - */ -public interface TbCoreDeviceRpcService { - - /** - * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. - * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} - * - * @param request the RPC request - * @param responseConsumer the consumer of the RPC response - */ - void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); - - /** - * Handles the RPC response from the Rule Engine. - * - * @param response the RPC response - */ - void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); - - /** - * Forwards the RPC request from Rule Engine to Device Actor - * - * @param request the RPC request message - */ - void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); - - /** - * Handles the RPC response from the Device Actor (Transport). - * - * @param response the RPC response - */ - void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index 3cfe9bc1f4..ba537aed25 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,8 +22,11 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import java.util.Optional; + /** * Created by ashvayka on 16.04.18. */ @@ -32,7 +35,7 @@ import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final String serviceId; + private final ServerAddress serverAddress; @Getter private final ToDeviceRpcRequest msg; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java similarity index 50% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java rename to application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java index f4bfefd878..570112dab3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java @@ -13,42 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.service.rpc; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import java.util.Objects; - +/** + * Created by ashvayka on 16.04.18. + */ @ToString -public class ServiceQueueKey { - @Getter - private final ServiceQueue serviceQueue; +@RequiredArgsConstructor +public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { @Getter private final TenantId tenantId; - public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { - this.serviceQueue = serviceQueue; - this.tenantId = tenantId; - } + @Getter + private final DeviceId deviceId; - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceQueueKey that = (ServiceQueueKey) o; - return serviceQueue.equals(that.serviceQueue) && - Objects.equals(tenantId, that.tenantId); - } + @Getter + private final ToServerRpcResponseMsg msg; @Override - public int hashCode() { - return Objects.hash(serviceQueue, tenantId); - } - - public ServiceType getServiceType() { - return serviceQueue.getType(); + public MsgType getMsgType() { + return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index daa74d013f..31827f542e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,53 +78,25 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); - protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); + switch (scriptType) { + case RULE_NODE_SCRIPT: + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); + default: + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - BlackListInfo errorCount = blackListedFunctions.get(scriptId); - if (errorCount != null) { - if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { - blackListedFunctions.remove(scriptId); - return false; - } else { - return errorCount.get() >= getMaxErrors(); - } + if (blackListedFunctions.containsKey(scriptId)) { + AtomicInteger errorCount = blackListedFunctions.get(scriptId); + return errorCount.get() >= getMaxErrors(); } else { return false; } } - - private class BlackListInfo { - private final AtomicInteger counter; - private long expirationTime; - - private BlackListInfo() { - this.counter = new AtomicInteger(0); - } - - public int get() { - return counter.get(); - } - - public int incrementAndGet() { - int result = counter.incrementAndGet(); - expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); - return result; - } - - public long getExpirationTime() { - return expirationTime; - } - } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index a9acfa4f42..49a6304ebb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 66a14cc827..909565609c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,8 +20,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.util.concurrent.TimeUnit; - @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -39,9 +37,6 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; - @Value("${js.local.max_black_list_duration_sec:60}") - private int maxBlackListDurationSec; - @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -61,9 +56,4 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } - - @Override - protected long getMaxBlacklistDuration() { - return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..1a218b119b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -23,13 +23,15 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaRequestTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -37,25 +39,42 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") +@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Value("${js.remote.request_topic}") + private String requestTopic; + + @Value("${js.remote.response_topic_prefix}") + private String responseTopicPrefix; + + @Value("${js.remote.max_pending_requests}") + private long maxPendingRequests; + @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; + @Value("${js.remote.response_poll_interval}") + private int responsePollDuration; + + @Value("${js.remote.response_auto_commit_interval}") + private int autoCommitInterval; + @Getter @Value("${js.remote.max_errors}") private int maxErrors; - @Value("${js.remote.max_black_list_duration_sec:60}") - private int maxBlackListDurationSec; - @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -80,20 +99,42 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - @Autowired - private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; - + private TbKafkaRequestTemplate kafkaTemplate; private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - requestTemplate.init(); + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(requestTopic); + requestBuilder.encoder(new RemoteJsRequestEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(new RemoteJsResponseDecoder()); + responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); + + TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder + builder = TbKafkaRequestTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.maxRequestTimeout(maxRequestsTimeout); + builder.pollInterval(responsePollDuration); + kafkaTemplate = builder.build(); + kafkaTemplate.init(); } @PreDestroy public void destroy() { - if (requestTemplate != null) { - requestTemplate.stop(); + if (kafkaTemplate != null) { + kafkaTemplate.stop(); } } @@ -110,12 +151,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); - + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback>() { + Futures.addCallback(future, new FutureCallback() { @Override - public void onSuccess(@Nullable TbProtoQueueMsg result) { + public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { kafkaEvalMsgs.incrementAndGet(); } @@ -128,7 +168,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -162,17 +202,16 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback>() { + Futures.addCallback(future, new FutureCallback() { @Override - public void onSuccess(@Nullable TbProtoQueueMsg result) { + public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { kafkaInvokeMsgs.incrementAndGet(); } @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -180,11 +219,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { - onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -202,8 +240,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); - JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + JsInvokeProtos.RemoteJsResponse response = future.get(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); @@ -214,9 +252,4 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - @Override - protected long getMaxBlacklistDuration() { - return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index 51ecba27b1..d07a2490ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,20 +17,19 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder> { +public class RemoteJsRequestEncoder implements TbKafkaEncoder { @Override - public byte[] encode(TbProtoQueueMsg value) { + public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { try { - return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 621d2b05d7..7a3876fb91 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,10 +16,8 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -27,12 +25,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder> { +public class RemoteJsResponseDecoder implements TbKafkaDecoder { @Override - public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { + public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); + return builder.build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 110f410fad..ef5d4716cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); + return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 572c3a6ed0..e359befb30 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -46,7 +46,6 @@ import ua_parser.Client; import java.util.UUID; - @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c13cb5529a..c58a664790 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.security.device; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsFilter; @@ -23,21 +24,16 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.util.TbCoreComponent; @Service -@TbCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - private final DeviceService deviceService; + @Autowired + DeviceService deviceService; - private final DeviceCredentialsService deviceCredentialsService; - - public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { - this.deviceService = deviceService; - this.deviceCredentialsService = deviceCredentialsService; - } + @Autowired + DeviceCredentialsService deviceCredentialsService; @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 5412ed6def..40d19da764 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.security.permission; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index a8c5dc13ae..9777af7b8e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -16,13 +16,20 @@ package org.thingsboard.server.service.security.permission; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 1b9d566206..952c63c9b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,9 +21,9 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Collections; +import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @@ -31,7 +31,6 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service -@TbCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 24eea18b53..99a24cfb12 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.state; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; @@ -22,13 +23,16 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -45,18 +49,16 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -67,6 +69,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -86,8 +89,8 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service -@TbCoreComponent @Slf4j +//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -101,15 +104,31 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - private final TenantService tenantService; - private final DeviceService deviceService; - private final AttributesService attributesService; - private final TimeseriesService tsService; - private final TbClusterService clusterService; - private final PartitionService partitionService; + @Autowired + private TenantService tenantService; + + @Autowired + private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + @Lazy + private ActorService actorService; + + @Autowired private TelemetrySubscriptionService tsSubService; + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService clusterRpcService; + @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -129,32 +148,18 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); - - public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, - AttributesService attributesService, TimeseriesService tsService, - TbClusterService clusterService, PartitionService partitionService) { - this.tenantService = tenantService; - this.deviceService = deviceService; - this.attributesService = attributesService; - this.tsService = tsService; - this.clusterService = clusterService; - this.partitionService = partitionService; - } - - @Autowired - public void setTsSubService(TelemetrySubscriptionService tsSubService) { - this.tsSubService = tsSubService; - } + private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); @PostConstruct public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); + queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); + //TODO: schedule persistence in v2.1; } @PreDestroy @@ -166,210 +171,133 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); + queueExecutor.submit(() -> onDeviceAddedSync(device)); } @Override public void onDeviceUpdated(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); + queueExecutor.submit(() -> onDeviceUpdatedSync(device)); } @Override - public void onDeviceDeleted(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); + public void onDeviceConnect(DeviceId deviceId) { + queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); } @Override - public void onDeviceConnect(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } + public void onDeviceActivity(DeviceId deviceId) { + deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); + queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); } @Override - public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { - deviceLastReportedActivity.put(deviceId, lastReportedActivity); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } + public void onDeviceDisconnect(DeviceId deviceId) { + queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); } @Override - public void onDeviceDisconnect(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } + public void onDeviceDeleted(Device device) { + queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } + queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); + } + + @Override + public void onClusterUpdate() { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(this::onClusterUpdateSync); } } @Override - public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { + public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.DeviceStateServiceMsgProto proto; try { - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - onDeviceDeleted(tenantId, deviceId); - callback.onSuccess(); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); - if (partitionedDevices.containsKey(tpi)) { - addDeviceUsingState(tpi, state); - callback.onSuccess(); - } else { - log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" - , tenantId, deviceId, tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - callback.onFailure(t); - } - }, MoreExecutors.directExecutor()); - } else if (proto.getUpdated()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } - } - } else { - //Device was probably deleted while message was in queue; - callback.onSuccess(); + proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + onDeviceAdded(device); + } else if (proto.getUpdated()) { + onDeviceUpdated(device); } } - } catch (Exception e) { - log.trace("Failed to process queue msg: [{}]", proto, e); - callback.onFailure(e); } } - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - synchronized (this) { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(() -> { - clusterUpdatePending = false; - initStateFromDB(partitionChangeEvent.getPartitions()); - }); + private void onClusterUpdateSync() { + clusterUpdatePending = false; + List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); + for (Tenant tenant : tenants) { + List> fetchFutures = new ArrayList<>(); + PageLink pageLink = new PageLink(initFetchPackSize); + while (pageLink != null) { + PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); + pageLink = page.hasNext() ? pageLink.nextPageLink() : null; + for (Device device : page.getData()) { + if (!routingService.resolveById(device.getId()).isPresent()) { + if (!deviceStates.containsKey(device.getId())) { + fetchFutures.add(fetchDeviceState(device)); + } + } else { + Set tenantDeviceSet = tenantDevices.get(tenant.getId()); + if (tenantDeviceSet != null) { + tenantDeviceSet.remove(device.getId()); + } + deviceStates.remove(device.getId()); + deviceLastReportedActivity.remove(device.getId()); + deviceLastSavedActivity.remove(device.getId()); + } + } + try { + Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + } catch (InterruptedException | ExecutionException e) { + log.warn("Failed to init device state service from DB", e); } } } } - private void initStateFromDB(Set partitions) { + private void initStateFromDB() { try { - Set addedPartitions = new HashSet<>(partitions); - addedPartitions.removeAll(partitionedDevices.keySet()); - - Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); - removedPartitions.removeAll(partitions); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set devices = partitionedDevices.remove(partition); - devices.forEach(deviceId -> { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - }); - }); - - addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); - - //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. - // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { + List> fetchFutures = new ArrayList<>(); PageLink pageLink = new PageLink(initFetchPackSize); while (pageLink != null) { - List> fetchFutures = new ArrayList<>(); PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.hasNext() ? pageLink.nextPageLink() : null; for (Device device : page.getData()) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); - if (addedPartitions.contains(tpi)) { - ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { - @Nullable - @Override - public Void apply(@Nullable DeviceStateData state) { - if (state != null) { - addDeviceUsingState(tpi, state); - } - return null; - } - }, MoreExecutors.directExecutor()); - fetchFutures.add(future); + if (!routingService.resolveById(device.getId()).isPresent()) { + fetchFutures.add(fetchDeviceState(device)); } } try { - Futures.successfulAsList(fetchFutures).get(); + Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } } } - log.info("Managing following partitions:"); - partitionedDevices.forEach((tpi, devices) -> { - log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); - }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } } - private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { - partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(DeviceStateData state) { + tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } @@ -397,24 +325,103 @@ public class DefaultDeviceStateService implements DeviceStateService { } } + private void onDeviceConnectSync(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } + } + + private void onDeviceDisconnectSync(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } + } + + private void onDeviceActivitySync(DeviceId deviceId) { + long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } + } + private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); + if (!routingService.resolveById(deviceId).isPresent()) { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); + } } } } return deviceStateData; } - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { - TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); + private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } + + private void onDeviceAddedSync(Device device) { + Optional address = routingService.resolveById(device.getId()); + if (!address.isPresent()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + addDeviceUsingState(state); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + } + }, MoreExecutors.directExecutor()); + } else { + sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); + } + } + + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { + log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); + ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -422,17 +429,40 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); + clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); + } + + private void onDeviceUpdatedSync(Device device) { + Optional address = routingService.resolveById(device.getId()); + if (!address.isPresent()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } else { + sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); + } } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - Set deviceIdSet = partitionedDevices.get(tpi); - deviceIdSet.remove(deviceId); + Optional address = routingService.resolveById(deviceId); + if (!address.isPresent()) { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + Set deviceIds = tenantDevices.get(tenantId); + if (deviceIds != null) { + deviceIds.remove(deviceId); + if (deviceIds.isEmpty()) { + tenantDevices.remove(tenantId); + } + } + } else { + sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); + } } private ListenableFuture fetchDeviceState(Device device) { @@ -493,9 +523,10 @@ public class DefaultDeviceStateService implements DeviceStateService { private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) { DeviceState state = stateData.getState(); try { - TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON - , json.writeValueAsString(state)); - clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON + , json.writeValueAsString(state) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } @@ -523,7 +554,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private static class AttributeSaveCallback implements FutureCallback { + private class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 1665a27b84..8dbe7579e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,17 +15,14 @@ */ package org.thingsboard.server.service.state; -import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.cluster.ServerAddress; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService extends ApplicationListener { +public interface DeviceStateService { void onDeviceAdded(Device device); @@ -35,12 +32,13 @@ public interface DeviceStateService extends ApplicationListener CALLBACK = new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to persist statistics", t); - } - }; - - private final TbServiceInfoProvider serviceInfoProvider; - private final TelemetrySubscriptionService tsService; - private final Lock lock = new ReentrantLock(); - private final AssetService assetService; - private final ConcurrentMap tenantQueueAssets; - - public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { - this.tsService = tsService; - this.serviceInfoProvider = serviceInfoProvider; - this.assetService = assetService; - this.tenantQueueAssets = new ConcurrentHashMap<>(); - } - - @Override - public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { - String queueName = ruleEngineStats.getQueueName(); - ruleEngineStats.getTenantStats().forEach((id, stats) -> { - TenantId tenantId = new TenantId(id); - try { - AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); - if (stats.getTotalMsgCounter().get() > 0) { - List tsList = stats.getCounters().entrySet().stream() - .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) - .collect(Collectors.toList()); - if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); - } - } - } catch (DataValidationException e) { - if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { - throw e; - } - } - }); - ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { - TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); - try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); - } catch (DataValidationException e2) { - if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { - throw e2; - } - } - }); - ruleEngineStats.reset(); - } - - private AssetId getServiceAssetId(TenantId tenantId, String queueName) { - TenantQueueKey key = new TenantQueueKey(tenantId, queueName); - AssetId assetId = tenantQueueAssets.get(key); - if (assetId == null) { - lock.lock(); - try { - assetId = tenantQueueAssets.get(key); - if (assetId == null) { - Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); - if (asset == null) { - asset = new Asset(); - asset.setTenantId(tenantId); - asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType(TB_SERVICE_QUEUE); - asset = assetService.saveAsset(asset); - } - assetId = asset.getId(); - tenantQueueAssets.put(key, assetId); - } - } finally { - lock.unlock(); - } - } - return assetId; - } - - @Data - private static class TenantQueueKey { - private final TenantId tenantId; - private final String queueName; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java deleted file mode 100644 index 88573844c9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.stats; - -import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; - -public interface RuleEngineStatisticsService { - - void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java deleted file mode 100644 index 67c81ac06f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ /dev/null @@ -1,380 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos.*; -import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Function; -import java.util.function.Predicate; - -@Slf4j -@TbCoreComponent -@Service -public class DefaultSubscriptionManagerService implements SubscriptionManagerService { - - @Autowired - private AttributesService attrService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TbServiceInfoProvider serviceInfoProvider; - - @Autowired - private TbQueueProducerProvider producerProvider; - - @Autowired - private TbLocalSubscriptionService localSubscriptionService; - - @Autowired - private DeviceStateService deviceStateService; - - @Autowired - private TbClusterService clusterService; - - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - - private ExecutorService tsCallBackExecutor; - private String serviceId; - private TbQueueProducer> toCoreNotificationsProducer; - - @PostConstruct - public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - serviceId = serviceInfoProvider.getServiceId(); - toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); - } - - @PreDestroy - public void shutdownExecutor() { - if (tsCallBackExecutor != null) { - tsCallBackExecutor.shutdownNow(); - } - } - - @Override - public void addSubscription(TbSubscription subscription, TbCallback callback) { - log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", - subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); - callback.onSuccess(); - } else { - log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" - , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); - } - boolean newSubscription = subscriptionsByEntityId - .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); - subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); - if (newSubscription) { - switch (subscription.getType()) { - case TIMESERIES: - handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); - break; - case ATTRIBUTES: - handleNewAttributeSubscription((TbAttributeSubscription) subscription); - break; - } - } - } - - @Override - public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - removeSubscriptionFromEntityMap(subscription); - removeSubscriptionFromPartitionMap(subscription); - if (sessionSubscriptions.isEmpty()) { - subscriptionsByWsSessionId.remove(sessionId); - } - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); - } - callback.onSuccess(); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - Set removedPartitions = new HashSet<>(currentPartitions); - removedPartitions.removeAll(partitionChangeEvent.getPartitions()); - - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set subs = partitionedSubscriptions.remove(partition); - if (subs != null) { - subs.forEach(this::removeSubscriptionFromEntityMap); - } - }); - } - } - - @Override - public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { - onLocalSubUpdate(entityId, - s -> { - if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { - return (TbTimeseriesSubscription) s; - } else { - return null; - } - }, s -> true, s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - callback.onSuccess(); - } - - @Override - public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { - onLocalSubUpdate(entityId, - s -> { - if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { - return (TbAttributeSubscription) s; - } else { - return null; - } - }, - s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), - s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - if (entityId.getEntityType() == EntityType.DEVICE) { - if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } - } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) - , null); - } - } - callback.onSuccess(); - } - - private void onLocalSubUpdate(EntityId entityId, - Function castFunction, - Predicate filterFunction, - Function> processFunction) { - Set entitySubscriptions = subscriptionsByEntityId.get(entityId); - if (entitySubscriptions != null) { - entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { - List subscriptionUpdate = processFunction.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - if (serviceId.equals(s.getServiceId())) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); - } else { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void removeSubscriptionFromEntityMap(TbSubscription sub) { - Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); - if (entitySubSet != null) { - entitySubSet.remove(sub); - if (entitySubSet.isEmpty()) { - subscriptionsByEntityId.remove(sub.getEntityId()); - } - } - } - - private void removeSubscriptionFromPartitionMap(TbSubscription sub) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); - Set subs = partitionedSubscriptions.get(tpi); - if (subs != null) { - subs.remove(sub); - } - } - - private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { - log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", - serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } - - private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { - log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", - serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().forEach((key, value) -> { - if (curTs > value) { - long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); - long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; - queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - - private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { - TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); - - builder.setSessionId(subscription.getSessionId()); - builder.setSubscriptionId(subscription.getSubscriptionId()); - - Map> data = new TreeMap<>(); - for (TsKvEntry tsEntry : updates) { - List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); - Object[] value = new Object[2]; - value[0] = tsEntry.getTs(); - value[1] = tsEntry.getValueAsString(); - values.add(value); - } - - data.forEach((key, value) -> { - TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); - dataBuilder.setKey(key); - value.forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - builder.addData(dataBuilder.build()); - }); - - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( - LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) - .build(); - return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java deleted file mode 100644 index d57067fa28..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entityview.EntityViewService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -@Slf4j -@TbCoreComponent -@Service -public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { - - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); - - @Autowired - private TelemetryWebSocketService wsService; - - @Autowired - private EntityViewService entityViewService; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TbClusterService clusterService; - - @Autowired - @Lazy - private SubscriptionManagerService subscriptionManagerService; - - private ExecutorService wsCallBackExecutor; - - @PostConstruct - public void initExecutor() { - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (wsCallBackExecutor != null) { - wsCallBackExecutor.shutdownNow(); - } - } - - @Override - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - } - } - - @Override - @EventListener(ClusterTopologyChangeEvent.class) - public void onApplicationEvent(ClusterTopologyChangeEvent event) { - if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { - /* - * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. - * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. - * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically - * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. - * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache - * Since number of subscriptions is usually much less then number of devices that are pushing data. - */ - subscriptionsBySessionId.values().forEach(map -> map.values() - .forEach(sub -> pushSubscriptionToManagerService(sub, false))); - } - } - - //TODO 3.1: replace null callbacks with callbacks from websocket service. - @Override - public void addSubscription(TbSubscription subscription) { - EntityId entityId = subscription.getEntityId(); - // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { - subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); - } - pushSubscriptionToManagerService(subscription, true); - registerSubscription(subscription); - } - - private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - // Subscription is managed on the same server; - if (pushToLocalService) { - subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); - } - } else { - // Push to the queue; - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); - clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); - } - } - - @Override - public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { - TbSubscription subscription = subscriptionsBySessionId - .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); - if (subscription != null) { - switch (subscription.getType()) { - case TIMESERIES: - TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; - update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); - break; - case ATTRIBUTES: - TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; - update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); - break; - } - wsService.sendWsMsg(sessionId, update); - } - callback.onSuccess(); - } - - @Override - public void cancelSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); - if (sessionSubscriptions != null) { - TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - if (sessionSubscriptions.isEmpty()) { - subscriptionsBySessionId.remove(sessionId); - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - // Subscription is managed on the same server; - subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); - } else { - // Push to the queue; - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); - clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); - } - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); - } - } - - @Override - public void cancelAllSessionSubscriptions(String sessionId) { - Map subscriptions = subscriptionsBySessionId.get(sessionId); - if (subscriptions != null) { - Set toRemove = new HashSet<>(subscriptions.keySet()); - toRemove.forEach(id -> cancelSubscription(sessionId, id)); - } - } - - private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); - - Map keyStates; - if (subscription.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = subscription.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - return TbTimeseriesSubscription.builder() - .serviceId(subscription.getServiceId()) - .sessionId(subscription.getSessionId()) - .subscriptionId(subscription.getSubscriptionId()) - .tenantId(subscription.getTenantId()) - .entityId(entityView.getEntityId()) - .startTime(entityView.getStartTimeMs()) - .endTime(entityView.getEndTimeMs()) - .allKeys(false) - .keyStates(keyStates).build(); - } - - private void registerSubscription(TbSubscription subscription) { - Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java deleted file mode 100644 index e20b707348..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbCallback; - -import java.util.List; - -public interface SubscriptionManagerService extends ApplicationListener { - - void addSubscription(TbSubscription subscription, TbCallback callback); - - void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); - - void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); - - void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java deleted file mode 100644 index 83a86efefc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.Builder; -import lombok.Data; -import lombok.Getter; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Map; - -public class TbAttributeSubscription extends TbSubscription { - - @Getter private final boolean allKeys; - @Getter private final Map keyStates; - @Getter private final TbAttributeSubscriptionScope scope; - - @Builder - public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, - boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { - super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); - this.allKeys = allKeys; - this.keyStates = keyStates; - this.scope = scope; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java deleted file mode 100644 index 58fba24250..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -public interface TbLocalSubscriptionService { - - void addSubscription(TbSubscription subscription); - - void cancelSubscription(String sessionId, int subscriptionId); - - void cancelAllSessionSubscriptions(String sessionId); - - void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); - - void onApplicationEvent(PartitionChangeEvent event); - - void onApplicationEvent(ClusterTopologyChangeEvent event); -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java deleted file mode 100644 index 22b37ff690..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Objects; - -@Data -@AllArgsConstructor -public abstract class TbSubscription { - - private final String serviceId; - private final String sessionId; - private final int subscriptionId; - private final TenantId tenantId; - private final EntityId entityId; - private final TbSubscriptionType type; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TbSubscription that = (TbSubscription) o; - return subscriptionId == that.subscriptionId && - sessionId.equals(that.sessionId) && - tenantId.equals(that.tenantId) && - entityId.equals(that.entityId) && - type == that.type; - } - - @Override - public int hashCode() { - return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java deleted file mode 100644 index d0b4d713f9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java +++ /dev/null @@ -1,247 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.UUID; - -public class TbSubscriptionUtils { - - public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() - .setServiceId(subscription.getServiceId()) - .setSessionId(subscription.getSessionId()) - .setSubscriptionId(subscription.getSubscriptionId()) - .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(subscription.getEntityId().getEntityType().name()) - .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); - - switch (subscription.getType()) { - case TIMESERIES: - TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; - TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() - .setSub(subscriptionProto) - .setAllKeys(tSub.isAllKeys()); - tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( - TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); - tSubProto.setStartTime(tSub.getStartTime()); - tSubProto.setEndTime(tSub.getEndTime()); - msgBuilder.setTelemetrySub(tSubProto.build()); - break; - case ATTRIBUTES: - TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; - TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() - .setSub(subscriptionProto) - .setAllKeys(aSub.isAllKeys()) - .setScope(aSub.getScope().name()); - aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( - TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); - msgBuilder.setAttributeSub(aSubProto.build()); - break; - } - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() - .setSessionId(subscription.getSessionId()) - .setSubscriptionId(subscription.getSubscriptionId()).build(); - msgBuilder.setSubClose(closeProto); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { - TbSubscriptionProto subProto = attributeSub.getSub(); - TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() - .serviceId(subProto.getServiceId()) - .sessionId(subProto.getSessionId()) - .subscriptionId(subProto.getSubscriptionId()) - .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) - .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); - - builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); - builder.allKeys(attributeSub.getAllKeys()); - Map keyStates = new HashMap<>(); - attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); - builder.keyStates(keyStates); - return builder.build(); - } - - public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { - TbSubscriptionProto subProto = telemetrySub.getSub(); - TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() - .serviceId(subProto.getServiceId()) - .sessionId(subProto.getSessionId()) - .subscriptionId(subProto.getSubscriptionId()) - .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) - .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); - - builder.allKeys(telemetrySub.getAllKeys()); - Map keyStates = new HashMap<>(); - telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); - builder.startTime(telemetrySub.getStartTime()); - builder.endTime(telemetrySub.getEndTime()); - builder.keyStates(keyStates); - return builder.build(); - } - - public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { - TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - msgBuilder.setTsUpdate(builder); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { - TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - msgBuilder.setAttrUpdate(builder); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { - KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); - switch (attr.getDataType()) { - case BOOLEAN: - attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); - break; - case LONG: - attr.getLongValue().ifPresent(dataBuilder::setLongV); - break; - case DOUBLE: - attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); - break; - case JSON: - attr.getJsonValue().ifPresent(dataBuilder::setJsonV); - break; - case STRING: - attr.getStrValue().ifPresent(dataBuilder::setStringV); - break; - } - return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); - } - - public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { - return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); - } - - public static List toTsKvEntityList(List dataList) { - List result = new ArrayList<>(dataList.size()); - dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); - return result; - } - - public static List toAttributeKvList(List dataList) { - List result = new ArrayList<>(dataList.size()); - dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); - return result; - } - - private static KvEntry getKvEntry(KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getType().getNumber()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongV()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStringV()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); - break; - } - return entry; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java deleted file mode 100644 index 0be63f7b65..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.Builder; -import lombok.Getter; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Map; - -public class TbTimeseriesSubscription extends TbSubscription { - - @Getter private final boolean allKeys; - @Getter private final Map keyStates; - @Getter private final long startTime; - @Getter private final long endTime; - - @Builder - public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, - boolean allKeys, Map keyStates, long startTime, long endTime) { - super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); - this.allKeys = allKeys; - this.keyStates = keyStates; - this.startTime = startTime; - this.endTime = endTime; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 665a787e6d..d29e5de5b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,43 +18,73 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.Subscription; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -63,36 +93,39 @@ import java.util.function.Consumer; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + @Autowired + private TelemetryWebSocketService wsService; - private final AttributesService attrService; - private final TimeseriesService tsService; - private final TbClusterService clusterService; - private final PartitionService partitionService; - private Optional subscriptionManagerService; + @Autowired + private AttributesService attrService; - private ExecutorService tsCallBackExecutor; - private ExecutorService wsCallBackExecutor; + @Autowired + private TimeseriesService tsService; - public DefaultTelemetrySubscriptionService(AttributesService attrService, - TimeseriesService tsService, - TbClusterService clusterService, - PartitionService partitionService) { - this.attrService = attrService; - this.tsService = tsService; - this.clusterService = clusterService; - this.partitionService = partitionService; - } + @Autowired + private ClusterRoutingService routingService; - @Autowired(required = false) - public void setSubscriptionManagerService(Optional subscriptionManagerService) { - this.subscriptionManagerService = subscriptionManagerService; - } + @Autowired + private ClusterRpcService rpcService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + @Lazy + private DeviceStateService stateService; + + @Autowired + @Lazy + private ActorService actorService; + + private ExecutorService tsCallBackExecutor; + private ExecutorService wsCallBackExecutor; @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); } @PreDestroy @@ -105,12 +138,64 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + @Override - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); + public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { + long startTime = 0L; + long endTime = 0L; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); + entityId = entityView.getEntityId(); + startTime = entityView.getStartTimeMs(); + endTime = entityView.getEndTimeMs(); + sub = getUpdatedSubscriptionState(entityId, sub, entityView); + } + Optional server = routingService.resolveById(entityId); + Subscription subscription; + if (server.isPresent()) { + ServerAddress address = server.get(); + log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); + subscription = new Subscription(sub, true, address, startTime, endTime); + tellNewSubscription(address, sessionId, subscription); + } else { + log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); + subscription = new Subscription(sub, true, null, startTime, endTime); + } + registerSubscription(sessionId, entityId, subscription); + } + + private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { + Map keyStates; + if (sub.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = sub.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); + } + + @Override + public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { + cleanupLocalWsSessionSubscriptions(sessionId); + } + + @Override + public void removeSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + Subscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); } } @@ -123,14 +208,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); } @Override @@ -157,31 +242,352 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - if (currentPartitions.contains(tpi)) { - if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); + @Override + public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + } + + @Override + public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionProto proto; + try { + proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + Map statesMap = proto.getKeyStatesList().stream().collect( + Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); + Subscription subscription = new Subscription( + new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), + new TenantId(UUID.fromString(proto.getTenantId())), + EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), + TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), + false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); + + addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); + } + + @Override + public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionUpdateProto proto; + try { + proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + SubscriptionUpdate update = convert(proto); + String sessionId = proto.getSessionId(); + log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); + Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); + if (subOpt.isPresent()) { + updateSubscriptionState(sessionId, subOpt.get(), update); + wsService.sendWsMsg(sessionId, update); + } + } + + @Override + public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionCloseProto proto; + try { + proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); + } + + @Override + public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SessionCloseProto proto; + try { + proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); + } + + @Override + public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.AttributeUpdateProto proto; + try { + proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), + proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); + } + + @Override + public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.TimeseriesUpdateProto proto; + try { + proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), + proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); + } + + @Override + public void onClusterUpdate() { + log.trace("Processing cluster onUpdate msg!"); + Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); + while (deviceIterator.hasNext()) { + Map.Entry> e = deviceIterator.next(); + Set subscriptions = e.getValue(); + Optional newAddressOptional = routingService.resolveById(e.getKey()); + if (newAddressOptional.isPresent()) { + newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); } else { - log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + checkSubscriptionsPrevAddress(subscriptions); + } + if (subscriptions.size() == 0) { + log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); + deviceIterator.remove(); + } + } + } + + private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { + Iterator subscriptionIterator = subscriptions.iterator(); + while (subscriptionIterator.hasNext()) { + Subscription s = subscriptionIterator.next(); + if (s.isLocal()) { + if (!newAddress.equals(s.getServer())) { + log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); + s.setServer(newAddress); + tellNewSubscription(newAddress, s.getWsSessionId(), s); + } + } else { + log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); + subscriptionIterator.remove(); + //TODO: onUpdate state of subscription by WsSessionId and other maps. + } + } + } + + private void checkSubscriptionsPrevAddress(Set subscriptions) { + for (Subscription s : subscriptions) { + if (s.isLocal() && s.getServer() != null) { + log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); + s.setServer(null); + } else { + log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); + } + } + } + + private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { + EntityId entityId = subscription.getEntityId(); + log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); + registerSubscription(sessionId, entityId, subscription); + if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().entrySet().forEach(e -> { + if (curTs > e.getValue()) { + queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); + } else { + log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + } + + private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { + Optional serverAddress = routingService.resolveById(entityId); + if (!serverAddress.isPresent()) { + onLocalAttributesUpdate(entityId, scope, attributes); + if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } } } else { - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); + tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); + } + } + + private void onTimeseriesUpdate(EntityId entityId, List ts) { + Optional serverAddress = routingService.resolveById(entityId); + if (!serverAddress.isPresent()) { + onLocalTimeseriesUpdate(entityId, ts); + } else { + tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); + } + } + + private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { + onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + } + + private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { + onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + } + + private boolean isInTimeRange(Subscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + if (deviceSubscriptions != null) { + deviceSubscriptions.stream().filter(filter).forEach(s -> { + String sessionId = s.getWsSessionId(); + List subscriptionUpdate = f.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + if (s.isLocal()) { + updateSubscriptionState(sessionId, s, update); + wsService.sendWsMsg(sessionId, update); + } else { + tellRemoteSubUpdate(s.getServer(), sessionId, update); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { + log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); + update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); + } + + private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { + Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); + deviceSubscriptions.add(subscription); + Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + + private void cleanupLocalWsSessionSubscriptions(String sessionId) { + cleanupWsSessionSubscriptions(sessionId, true); + } + + private void cleanupRemoteWsSessionSubscriptions(String sessionId) { + cleanupWsSessionSubscriptions(sessionId, false); + } + + private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { + log.debug("[{}] Removing all subscriptions for particular session.", sessionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + int sessionSubscriptionSize = sessionSubscriptions.size(); + + for (Subscription subscription : sessionSubscriptions.values()) { + EntityId entityId = subscription.getEntityId(); + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + deviceSubscriptions.remove(subscription); + if (deviceSubscriptions.isEmpty()) { + subscriptionsByEntityId.remove(entityId); + } + } + subscriptionsByWsSessionId.remove(sessionId); + log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); + + if (localSession) { + notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); + } + } else { + log.debug("[{}] No subscriptions found!", sessionId); + } + } + + private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { + Set affectedServers = new HashSet<>(); + for (Subscription subscription : sessionSubscriptions.values()) { + if (subscription.getServer() != null) { + affectedServers.add(subscription.getServer()); + } + } + for (ServerAddress address : affectedServers) { + log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); + tellRemoteSessionClose(address, sessionId); } } - private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - if (currentPartitions.contains(tpi)) { - if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); + private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { + EntityId entityId = subscription.getEntityId(); + if (subscription.isLocal() && subscription.getServer() != null) { + tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); + } + if (sessionSubscriptions.isEmpty()) { + log.debug("[{}] Removed last subscription for particular session.", sessionId); + subscriptionsByWsSessionId.remove(sessionId); + } else { + log.debug("[{}] Removed session subscription.", sessionId); + } + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + if (deviceSubscriptions != null) { + boolean result = deviceSubscriptions.remove(subscription); + if (result) { + if (deviceSubscriptions.size() == 0) { + log.debug("[{}] Removed last subscription for particular device.", sessionId); + subscriptionsByEntityId.remove(entityId); + } else { + log.debug("[{}] Removed device subscription.", sessionId); + } } else { - log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + log.debug("[{}] Subscription not found!", sessionId); } } else { - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); + log.debug("[{}] No device subscriptions found!", sessionId); } } @@ -211,4 +617,161 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } + + private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { + ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setSubscriptionId(sub.getSubscriptionId()); + builder.setTenantId(sub.getSub().getTenantId().getId().toString()); + builder.setEntityType(sub.getEntityId().getEntityType().name()); + builder.setEntityId(sub.getEntityId().getId().toString()); + builder.setType(sub.getType().name()); + builder.setAllKeys(sub.isAllKeys()); + if (sub.getScope() != null) { + builder.setScope(sub.getScope()); + } + sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( + ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { + ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setSubscriptionId(update.getSubscriptionId()); + builder.setErrorCode(update.getErrorCode()); + if (update.getErrorMsg() != null) { + builder.setErrorMsg(update.getErrorMsg()); + } + update.getData().entrySet().forEach( + e -> { + ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); + + dataBuilder.setKey(e.getKey()); + e.getValue().forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + + builder.addData(dataBuilder.build()); + } + ); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { + ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); + builder.setEntityId(entityId.getId().toString()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { + ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); + builder.setEntityId(entityId.getId().toString()); + builder.setEntityType(entityId.getEntityType().name()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteSessionClose(ServerAddress address, String sessionId) { + ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); + } + + private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { + ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); + } + + private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { + ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setTs(ts); + dataBuilder.setValueType(attr.getDataType().ordinal()); + switch (attr.getDataType()) { + case BOOLEAN: + Optional booleanValue = attr.getBooleanValue(); + booleanValue.ifPresent(dataBuilder::setBoolValue); + break; + case LONG: + Optional longValue = attr.getLongValue(); + longValue.ifPresent(dataBuilder::setLongValue); + break; + case DOUBLE: + Optional doubleValue = attr.getDoubleValue(); + doubleValue.ifPresent(dataBuilder::setDoubleValue); + break; + case JSON: + Optional jsonValue = attr.getJsonValue(); + jsonValue.ifPresent(dataBuilder::setJsonValue); + break; + case STRING: + Optional stringValue = attr.getStrValue(); + stringValue.ifPresent(dataBuilder::setStrValue); + break; + } + return dataBuilder; + } + + private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { + return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); + } + + private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { + return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); + } + + private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getValueType()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); + break; + } + return entry; + } + + private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + private Optional getSubscription(String sessionId, int subscriptionId) { + Subscription state = null; + Map subMap = subscriptionsByWsSessionId.get(sessionId); + if (subMap != null) { + state = subMap.get(subscriptionId); + } + return Optional.ofNullable(state); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 6afcaf97bb..741efe6f8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,18 +43,12 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; -import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; -import org.thingsboard.server.service.subscription.TbAttributeSubscription; -import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; @@ -63,6 +57,7 @@ import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -88,7 +83,6 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service -@TbCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { @@ -104,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TbLocalSubscriptionService subService; + private TelemetrySubscriptionService subscriptionManager; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -118,9 +112,6 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; - @Autowired - private TbServiceInfoProvider serviceInfoProvider; - @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -136,11 +127,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; - private String serviceId; @PostConstruct public void initExecutor() { - serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -164,7 +153,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subService.cancelAllSessionSubscriptions(sessionId); + subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); processSessionClose(sessionRef); break; } @@ -345,18 +334,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); - - TbAttributeSubscription sub = TbAttributeSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(false) - .keyStates(subState) - .scope(scope).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -442,18 +421,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); - - TbAttributeSubscription sub = TbAttributeSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(true) - .keyStates(subState) - .scope(scope).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -525,16 +494,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - - TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(true) - .keyStates(subState).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -559,19 +520,12 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); + Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - - TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(false) - .keyStates(subState).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -590,9 +544,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subService.cancelAllSessionSubscriptions(sessionId); + subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); } else { - subService.cancelSubscription(sessionId, cmd.getCmdId()); + subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d652a3f1c5..d77a0e50dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,13 +15,34 @@ */ package org.thingsboard.server.service.telemetry; -import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { + void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); + + void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); + + void removeSubscription(String sessionId, int cmdId); + + void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); + + void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); + + void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); + + void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); + + void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); + + void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); + + void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java new file mode 100644 index 0000000000..7f7db9430d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry.sub; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.telemetry.TelemetryFeature; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class Subscription { + + private final SubscriptionState sub; + private final boolean local; + private ServerAddress server; + private long startTime; + private long endTime; + + public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { + this(sub, local, server, 0L, 0L); + } + + public String getWsSessionId() { + return getSub().getWsSessionId(); + } + + public int getSubscriptionId() { + return getSub().getSubscriptionId(); + } + + public EntityId getEntityId() { + return getSub().getEntityId(); + } + + public TelemetryFeature getType() { + return getSub().getType(); + } + + public String getScope() { + return getSub().getScope(); + } + + public boolean isAllKeys() { + return getSub().isAllKeys(); + } + + public Map getKeyStates() { + return getSub().getKeyStates(); + } + + public void setKeyState(String key, long ts) { + getSub().getKeyStates().put(key, ts); + } + + @Override + public String toString() { + return "Subscription{" + + "sub=" + sub + + ", local=" + local + + ", server=" + server + + '}'; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 992dc26af9..68aaca34c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { + return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java new file mode 100644 index 0000000000..1aee532c93 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -0,0 +1,242 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transaction; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.executors.DbCallbackExecutorService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +@Service +@Slf4j +public class BaseRuleChainTransactionService implements RuleChainTransactionService { + + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService clusterRpcService; + + @Autowired + private DbCallbackExecutorService callbackExecutor; + + @Value("${actors.rule.transaction.queue_size}") + private int finalQueueSize; + @Value("${actors.rule.transaction.duration}") + private long duration; + + private final Lock transactionLock = new ReentrantLock(); + private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); + private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); + + private ExecutorService timeoutExecutor; + + @PostConstruct + public void init() { + timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); + executeOnTimeout(); + } + + @PreDestroy + public void destroy() { + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + } + + @Override + public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { + transactionLock.lock(); + try { + BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> + new LinkedBlockingQueue<>(finalQueueSize)); + + TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); + int queueSize = queue.size(); + if (queueSize >= finalQueueSize) { + log.trace("Queue has no space: {}", transactionTask); + executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); + } else { + addMsgToQueues(queue, transactionTask); + if (queueSize == 0) { + executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); + } else { + log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); + } + } + } finally { + transactionLock.unlock(); + } + } + + @Override + public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { + Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); + if (address.isPresent()) { + sendTransactionEventToRemoteServer(msg, address.get()); + executeOnSuccess(onSuccess, msg); + } else { + endLocalTransaction(msg, onSuccess, onFailure); + } + } + + @Override + public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { + endLocalTransaction(TbMsg.fromBytes(data), msg -> { + }, error -> { + }); + } + + private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { + queue.offer(transactionTask); + timeoutQueue.offer(transactionTask); + log.trace("Added msg to queue, size: [{}]", queue.size()); + } + + private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { + transactionLock.lock(); + try { + BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> + new LinkedBlockingQueue<>(finalQueueSize)); + + TbTransactionTask currentTransactionTask = queue.peek(); + if (currentTransactionTask != null) { + if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { + currentTransactionTask.setCompleted(true); + queue.poll(); + log.trace("Removed msg from queue, size [{}]", queue.size()); + + executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); + executeOnSuccess(onSuccess, msg); + + TbTransactionTask nextTransactionTask = queue.peek(); + if (nextTransactionTask != null) { + executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); + } + } else { + log.trace("Task has expired!"); + executeOnFailure(onFailure, "Task has expired!"); + } + } else { + log.trace("Queue is empty, previous task has expired!"); + executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); + } + } finally { + transactionLock.unlock(); + } + } + + private void executeOnTimeout() { + timeoutExecutor.submit(() -> { + while (true) { + TbTransactionTask transactionTask = timeoutQueue.peek(); + if (transactionTask != null) { + long sleepDuration = 0L; + transactionLock.lock(); + try { + if (transactionTask.isCompleted()) { + timeoutQueue.poll(); + } else { + long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); + if (expIn < 0) { + log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); + timeoutQueue.poll(); + executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); + + BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); + if (queue != null) { + queue.poll(); + TbTransactionTask nextTransactionTask = queue.peek(); + if (nextTransactionTask != null) { + executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); + } + } + } else { + sleepDuration = Math.min(expIn, duration); + } + } + } finally { + transactionLock.unlock(); + } + if (sleepDuration > 0L) { + try { + log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); + TimeUnit.MILLISECONDS.sleep(sleepDuration); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted", e); + } + } + } else { + try { + log.trace("Queue is empty, waiting for tasks!"); + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted", e); + } + } + } + }); + } + + private void executeOnFailure(Consumer onFailure, String exception) { + executeCallback(() -> { + onFailure.accept(new RuntimeException(exception)); + return null; + }); + } + + private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { + executeCallback(() -> { + onSuccess.accept(tbMsg); + return null; + }); + } + + private void executeCallback(Callable task) { + callbackExecutor.executeAsync(task); + } + + private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { + log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); + clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java new file mode 100644 index 0000000000..c86f882ff7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transaction; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.function.Consumer; + +@Data +@AllArgsConstructor +public final class TbTransactionTask { + + private final TbMsg msg; + private final Consumer onStart; + private final Consumer onEnd; + private final Consumer onFailure; + private final long expirationTime; + + private boolean isCompleted; + + public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { + this.msg = msg; + this.onStart = onStart; + this.onEnd = onEnd; + this.onFailure = onFailure; + this.expirationTime = expirationTime; + this.isCompleted = false; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java deleted file mode 100644 index c9e37e6791..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; -import java.util.function.Consumer; - -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; - -@Slf4j -@Service -@TbCoreComponent -public class DefaultTbCoreToTransportService implements TbCoreToTransportService { - - private final PartitionService partitionService; - private final TbQueueProducer> tbTransportProducer; - - public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { - this.partitionService = partitionService; - this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); - } - - @Override - public void process(String nodeId, ToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); - tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private static class QueueCallbackAdaptor implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java index 7456ca7662..8e820fbffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -34,19 +33,14 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -58,15 +52,10 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -@TbCoreComponent -public class DefaultTransportApiService implements TransportApiService { +public class LocalTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); - //TODO: Constructor dependencies; - @Autowired - private TenantService tenantService; - @Autowired private DeviceService deviceService; @@ -85,20 +74,17 @@ public class DefaultTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { - TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); + public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); - } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); } - return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return getEmptyTransportApiResponseFuture(); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -139,13 +125,6 @@ public class DefaultTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } - private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { - TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); - ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); - return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() - .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) - .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); - } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java new file mode 100644 index 0000000000..32ff77877d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java @@ -0,0 +1,227 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.dao.device.ClaimDevicesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 12.10.18. + */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") +public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { + + @Autowired + private TransportApiService transportApiService; + + @Autowired + private ActorSystemContext actorContext; + + //TODO: completely replace this routing with the Kafka routing by partition ids. + @Autowired + private ClusterRoutingService routingService; + @Autowired + private ClusterRpcService rpcService; + @Autowired + private DataDecodingEncodingService encodingService; + @Autowired + private ClaimDevicesService claimDevicesService; + + @PostConstruct + public void init() { + super.init(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); + } + + @Override + public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); + } + + @Override + protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { + TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); + + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + callback.onSuccess(null); + } else { + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); + DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), + callback::onSuccess, callback::onError); + } + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + processToTransportMsg(msg); + if (onSuccess != null) { + onSuccess.run(); + } + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + } else { + actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); + } + if (callback != null) { + callback.onSuccess(null); + } + } + + private Consumer getThrowableConsumer(TransportServiceCallback callback) { + return e -> { + if (callback != null) { + callback.onError(e); + } + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java new file mode 100644 index 0000000000..7aa438c64f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -0,0 +1,251 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import akka.actor.ActorRef; +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.BlockingBucket; +import io.github.bucket4j.Bucket4j; +import io.github.bucket4j.local.LocalBucket; +import io.github.bucket4j.local.LocalBucketBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 09.10.18. + */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +public class RemoteRuleEngineTransportService implements RuleEngineTransportService { + + @Value("${transport.remote.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${transport.remote.notifications.topic}") + private String notificationsTopic; + @Value("${transport.remote.rule_engine.poll_interval}") + private int pollDuration; + @Value("${transport.remote.rule_engine.auto_commit_interval}") + private int autoCommitInterval; + + @Value("${transport.remote.rule_engine.poll_records_pack_size}") + private int pollRecordsPackSize; + @Value("${transport.remote.rule_engine.max_poll_records_per_second}") + private long pollRecordsPerSecond; + @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") + private long pollRecordsPerMinute; + @Value("${transport.remote.rule_engine.stats.enabled:false}") + private boolean statsEnabled; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private ActorSystemContext actorContext; + + //TODO: completely replace this routing with the Kafka routing by partition ids. + @Autowired + private ClusterRoutingService routingService; + @Autowired + private ClusterRpcService rpcService; + @Autowired + private DataDecodingEncodingService encodingService; + + private TBKafkaConsumerTemplate ruleEngineConsumer; + private TBKafkaProducerTemplate notificationsProducer; + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); + + private volatile boolean stopped = false; + + private final RuleEngineStats stats = new RuleEngineStats(); + + @PostConstruct + public void init() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); + notificationsProducerBuilder.settings(kafkaSettings); + notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); + notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); + + notificationsProducer = notificationsProducerBuilder.build(); + notificationsProducer.init(); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); + ruleEngineConsumerBuilder.settings(kafkaSettings); + ruleEngineConsumerBuilder.topic(ruleEngineTopic); + ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); + ruleEngineConsumerBuilder.groupId("tb-node"); + ruleEngineConsumerBuilder.autoCommit(true); + ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); + ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); + ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); + + ruleEngineConsumer = ruleEngineConsumerBuilder.build(); + ruleEngineConsumer.subscribe(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + LocalBucketBuilder builder = Bucket4j.builder(); + builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); + builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); + LocalBucket pollRateBucket = builder.build(); + BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); + int recordsCount = records.count(); + if (recordsCount > 0) { + while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { + log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); + Thread.sleep(TimeUnit.SECONDS.toMillis(1)); + } + log.trace("Processing {} records", recordsCount); + } + records.forEach(record -> { + try { + ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + String topic = notificationsTopic + "." + nodeId; + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); + log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + } else { + log.trace("Pushing message to local server: {}", toDeviceActorMsg); + actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (ruleEngineConsumer != null) { + ruleEngineConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + + private static class QueueCallbackAdaptor implements Callback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + if (onSuccess != null) { + onSuccess.run(); + } + } else { + if (onFailure != null) { + onFailure.accept(exception); + } + } + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java new file mode 100644 index 0000000000..0c9efc7d1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -0,0 +1,111 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.*; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 05.10.18. + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +public class RemoteTransportApiService { + + @Value("${transport.remote.transport_api.requests_topic}") + private String transportApiRequestsTopic; + @Value("${transport.remote.transport_api.max_pending_requests}") + private int maxPendingRequests; + @Value("${transport.remote.transport_api.request_timeout}") + private long requestTimeout; + @Value("${transport.remote.transport_api.request_poll_interval}") + private int responsePollDuration; + @Value("${transport.remote.transport_api.request_auto_commit_interval}") + private int autoCommitInterval; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private TransportApiService transportApiService; + + private ExecutorService transportCallbackExecutor; + + private TbKafkaResponseTemplate transportApiTemplate; + + @PostConstruct + public void init() { + this.transportCallbackExecutor = Executors.newWorkStealingPool(100); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); + responseBuilder.encoder(new TransportApiResponseEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.topic(transportApiRequestsTopic); + requestBuilder.clientId(nodeIdProvider.getNodeId()); + requestBuilder.groupId("tb-node"); + requestBuilder.autoCommit(true); + requestBuilder.autoCommitIntervalMs(autoCommitInterval); + requestBuilder.decoder(new TransportApiRequestDecoder()); + + TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder + builder = TbKafkaResponseTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.requestTimeout(requestTimeout); + builder.pollInterval(responsePollDuration); + builder.executor(transportCallbackExecutor); + builder.handler(transportApiService); + transportApiTemplate = builder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + transportApiTemplate.init(); + } + + @PreDestroy + public void destroy() { + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java rename to application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java index c362c5e889..dea6e6d33c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,26 +21,31 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class TbCoreConsumerStats { +public class RuleEngineStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); + private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); + private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); + private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); - private final AtomicInteger deviceStateCounter = new AtomicInteger(0); - private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0); - private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0); - public void log(TransportProtos.TransportToDeviceActorMsg msg) { totalCounter.incrementAndGet(); if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } + if (msg.hasPostTelemetry()) { + postTelemetryCounter.incrementAndGet(); + } + if (msg.hasPostAttributes()) { + postAttributesCounter.incrementAndGet(); + } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -53,6 +58,9 @@ public class TbCoreConsumerStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } + if (msg.hasToServerRPCCallRequest()) { + toServerRPCCallRequestCounter.incrementAndGet(); + } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } @@ -61,32 +69,15 @@ public class TbCoreConsumerStats { } } - public void log(TransportProtos.DeviceStateServiceMsgProto msg) { - totalCounter.incrementAndGet(); - deviceStateCounter.incrementAndGet(); - } - - public void log(TransportProtos.SubscriptionMgrMsgProto msg) { - totalCounter.incrementAndGet(); - subscriptionMsgCounter.incrementAndGet(); - } - - public void log(TransportProtos.ToCoreNotificationMsg msg) { - totalCounter.incrementAndGet(); - toCoreNotificationsCounter.incrementAndGet(); - } - public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + - " deviceState [{}] subMgr [{}] coreNfs [{}]", - total, sessionEventCounter.getAndSet(0), - getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + + "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", + total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), + postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) - , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); + toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); } } - } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java new file mode 100644 index 0000000000..0d401e4d9c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; + +import java.util.function.Consumer; + +/** + * Created by ashvayka on 05.10.18. + */ +public interface RuleEngineTransportService { + + void process(String nodeId, DeviceActorToTransportMsg msg); + + void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java deleted file mode 100644 index 566bf99cd3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 05.10.18. - */ -@Slf4j -@Service -@TbCoreComponent -public class TbCoreTransportApiService { - - private final TbCoreQueueFactory tbCoreQueueFactory; - private final TransportApiService transportApiService; - - @Value("${queue.transport_api.max_pending_requests:10000}") - private int maxPendingRequests; - @Value("${queue.transport_api.max_requests_timeout:10000}") - private long requestTimeout; - @Value("${queue.transport_api.request_poll_interval:25}") - private int responsePollDuration; - @Value("${queue.transport_api.max_callback_threads:100}") - private int maxCallbackThreads; - - private ExecutorService transportCallbackExecutor; - private TbQueueResponseTemplate, - TbProtoQueueMsg> transportApiTemplate; - - public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { - this.tbCoreQueueFactory = tbCoreQueueFactory; - this.transportApiService = transportApiService; - } - - @PostConstruct - public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); - TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); - TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); - - DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); - builder.requestTemplate(consumer); - builder.responseTemplate(producer); - builder.maxPendingRequests(maxPendingRequests); - builder.requestTimeout(requestTimeout); - builder.pollInterval(responsePollDuration); - builder.executor(transportCallbackExecutor); - builder.handler(transportApiService); - transportApiTemplate = builder.build(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); - } - - @PreDestroy - public void destroy() { - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java similarity index 64% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java rename to application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index 2b2c203ec5..9f08463f91 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.kafka.TbKafkaDecoder; -import java.util.UUID; +import java.io.IOException; -public class IdMsgPair { - final UUID uuid; - final TbProtoQueueMsg msg; - - public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { - this.uuid = uuid; - this.msg = msg; +/** + * Created by ashvayka on 05.10.18. + */ +public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { + @Override + public ToRuleEngineMsg decode(byte[] data) throws IOException { + return ToRuleEngineMsg.parseFrom(data); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java rename to application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java index e0d5e2121a..9c3f5634bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java @@ -16,13 +16,14 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.kafka.TbKafkaEncoder; -import java.util.function.Consumer; - -public interface TbCoreToTransportService { - - void process(String nodeId, ToTransportMsg msg); - - void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); - +/** + * Created by ashvayka on 05.10.18. + */ +public class ToTransportMsgEncoder implements TbKafkaEncoder { + @Override + public byte[] encode(ToTransportMsg value) { + return value.toByteArray(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java new file mode 100644 index 0000000000..ac1c5965f3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.kafka.TbKafkaDecoder; + +import java.io.IOException; + +/** + * Created by ashvayka on 05.10.18. + */ +public class TransportApiRequestDecoder implements TbKafkaDecoder { + @Override + public TransportApiRequestMsg decode(byte[] data) throws IOException { + return TransportApiRequestMsg.parseFrom(data); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java new file mode 100644 index 0000000000..35f8a7b1aa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.kafka.TbKafkaEncoder; + +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Created by ashvayka on 05.10.18. + */ +public class TransportApiResponseEncoder implements TbKafkaEncoder { + @Override + public byte[] encode(TransportApiResponseMsg value) { + return value.toByteArray(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index c9edfa9331..2964934313 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,13 +15,11 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.queue.TbQueueHandler; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.kafka.TbKafkaHandler; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { +public interface TransportApiService extends TbKafkaHandler { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index ba4e7baab1..34abac72bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -37,11 +36,9 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; - private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { this.msg = msg; - this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java index 61d81ae0b0..2399450d86 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java @@ -17,27 +17,47 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.server.dao.util.PsqlDao; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; - +@PsqlTsAnyDao @Slf4j -@PsqlDao -public abstract class AbstractCleanUpService { +public abstract class AbstractTimeseriesCleanUpService { + + @Value("${sql.ttl.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.enabled}") + private boolean ttlTaskExecutionEnabled; @Value("${spring.datasource.url}") - protected String dbUrl; + private String dbUrl; @Value("${spring.datasource.username}") - protected String dbUserName; + private String dbUserName; @Value("${spring.datasource.password}") - protected String dbPassword; + private String dbPassword; + + @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + protected abstract void doCleanUp(Connection connection); protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -54,7 +74,7 @@ public abstract class AbstractCleanUpService { return removed; } - protected void getWarnings(Statement statement) throws SQLException { + private void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -66,6 +86,4 @@ public abstract class AbstractCleanUpService { } } - protected abstract void doCleanUp(Connection connection); - -} +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java index cd403ee3b8..ab344dc518 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl.timeseries; +package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java index f5898b9b20..1dbdb4ad55 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl.timeseries; +package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java deleted file mode 100644 index a608ca257b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.ttl.events; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.service.ttl.AbstractCleanUpService; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -@PsqlDao -@Slf4j -@Service -public class EventsCleanUpService extends AbstractCleanUpService { - - @Value("${sql.ttl.events.events_ttl}") - private long ttl; - - @Value("${sql.ttl.events.debug_events_ttl}") - private long debugTtl; - - @Value("${sql.ttl.events.enabled}") - private boolean ttlTaskExecutionEnabled; - - @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - @Override - protected void doCleanUp(Connection connection) { - long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); - log.info("Total events removed by TTL: [{}]", totalEventsRemoved); - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java deleted file mode 100644 index 75b07b9176..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.ttl.timeseries; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; -import org.thingsboard.server.service.ttl.AbstractCleanUpService; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { - - @Value("${sql.ttl.ts.ts_key_value_ttl}") - protected long systemTtl; - - @Value("${sql.ttl.ts.enabled}") - private boolean ttlTaskExecutionEnabled; - - @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 4b7f0f6366..1b8173c424 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,7 +24,6 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; -import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -39,7 +38,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service -@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java b/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java similarity index 97% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java rename to application/src/main/java/org/thingsboard/server/utils/JsonUtils.java index 583b73dba1..5621362c09 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.utils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto new file mode 100644 index 0000000000..cfacc66121 --- /dev/null +++ b/application/src/main/proto/cluster.proto @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package cluster; + +option java_package = "org.thingsboard.server.gen.cluster"; +option java_outer_classname = "ClusterAPIProtos"; + +service ClusterRpcService { + rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} +} + +message ClusterMessage { + MessageType messageType = 1; + MessageMataInfo messageMetaInfo = 2; + ServerAddress serverAddress = 3; + bytes payload = 4; +} + +message ServerAddress { + string host = 1; + int32 port = 2; +} + +message MessageMataInfo { + string payloadMetaInfo = 1; + repeated string tags = 2; +} + +enum MessageType { + + //Cluster control messages + RPC_SESSION_CREATE_REQUEST_MSG = 0; + TO_ALL_NODES_MSG = 1; + RPC_SESSION_TELL_MSG = 2; + RPC_BROADCAST_MSG = 3; + CONNECT_RPC_MESSAGE =4; + + CLUSTER_ACTOR_MESSAGE = 5; + // Messages related to TelemetrySubscriptionService + CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; + CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; + CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; + CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; + CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; + CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; + CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; + + CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; + CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; +} + +// Messages related to CLUSTER_TELEMETRY_MESSAGE +message SubscriptionProto { + string sessionId = 1; + int32 subscriptionId = 2; + string entityType = 3; + string tenantId = 4; + string entityId = 5; + string type = 6; + bool allKeys = 7; + repeated SubscriptionKetStateProto keyStates = 8; + string scope = 9; +} + +message SubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated SubscriptionUpdateValueListProto data = 5; +} + +message AttributeUpdateProto { + string entityType = 1; + string entityId = 2; + string scope = 3; + repeated KeyValueProto data = 4; +} + +message TimeseriesUpdateProto { + string entityType = 1; + string entityId = 2; + repeated KeyValueProto data = 4; +} + +message SessionCloseProto { + string sessionId = 1; +} + +message SubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message SubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message SubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + +message KeyValueProto { + string key = 1; + int64 ts = 2; + int32 valueType = 3; + string strValue = 4; + int64 longValue = 5; + double doubleValue = 6; + bool boolValue = 7; + string jsonValue = 8; +} + +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} diff --git a/common/queue/src/main/proto/jsinvoke.proto b/application/src/main/proto/jsinvoke.proto similarity index 100% rename from common/queue/src/main/proto/jsinvoke.proto rename to application/src/main/proto/jsinvoke.proto diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 614f85ba0e..e147325f42 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,9 +28,6 @@ - - - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c896b3fc85..8dc7089bbe 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -70,7 +70,21 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" +# RPC connection parameters. Used only in cluster mode only. +rpc: + bind_host: "${RPC_HOST:localhost}" + bind_port: "${RPC_PORT:9001}" + +# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: + # Unique id for this node (autogenerated if empty) + node_id: "${CLUSTER_NODE_ID:}" + # Name of hash function used for consistent hash ring. + hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" + # Amount of virtual nodes in consistent hash ring. + vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" + # Queue partition id for current node + partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -201,18 +215,14 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - ts: - enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds - events: - enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week + enabled: "${SQL_TTL_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds + ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: + cluster: + grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -255,6 +265,8 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -334,19 +346,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -375,7 +387,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" # Audit log parameters audit-log: @@ -415,11 +427,32 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - # Should be greater then transport.sessions.report_timeout - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -429,27 +462,55 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" - # Maximum time in seconds for black listed function to stay in the list. - max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" - # Maximum time in seconds for black listed function to stay in the list. - max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: + type: "${TRANSPORT_TYPE:local}" # local or remote + remote: + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" + poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" + max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" + max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" + stats: + enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -462,8 +523,6 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -507,7 +566,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:ThingsBoard REST API}" + title: "${SWAGGER_TITLE:Thingsboard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -517,160 +576,3 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" - -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" - transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" - poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. - diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 1e445adf3c..bed5689ceb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,7 +33,6 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -116,7 +115,9 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = MediaType.APPLICATION_JSON; + protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); protected MockMvc mockMvc; @@ -199,7 +200,6 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); - log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index c74c83dfe8..23e23c31fe 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -72,7 +71,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -112,27 +111,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 3; i++) { + for (int i=0;i<3;i++) { Asset asset = new Asset(); - asset.setName("My asset B" + i); + asset.setName("My asset B"+i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i = 0; i < 7; i++) { + for (int i=0;i<7;i++) { Asset asset = new Asset(); - asset.setName("My asset C" + i); + asset.setName("My asset C"+i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i = 0; i < 9; i++) { + for (int i=0;i<9;i++) { Asset asset = new Asset(); - asset.setName("My asset A" + i); + asset.setName("My asset A"+i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>() { - }); + new TypeReference>(){}); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -148,10 +146,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/" + savedAsset.getId().getId().toString()) + doDelete("/api/asset/"+savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/" + savedAsset.getId().getId().toString()) + doGet("/api/asset/"+savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -246,16 +244,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 178; i++) { + for (int i=0;i<178;i++) { Asset asset = new Asset(); - asset.setName("Asset" + i); + asset.setName("Asset"+i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -271,8 +269,6 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); - loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); - Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -283,10 +279,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -294,10 +290,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i = 0; i < 75; i++) { + for (int i=0;i<75;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -338,7 +334,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -349,7 +345,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -365,10 +361,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -377,10 +373,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i = 0; i < 75; i++) { + for (int i=0;i<75;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -421,7 +417,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -432,7 +428,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -451,9 +447,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i = 0; i < 128; i++) { + for (int i=0;i<128;i++) { Asset asset = new Asset(); - asset.setName("Asset" + i); + asset.setName("Asset"+i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -487,10 +483,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i = 0; i < 125; i++) { + for (int i=0;i<125;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -500,10 +496,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -578,10 +574,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i = 0; i < 125; i++) { + for (int i=0;i<125;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -592,10 +588,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 34c39bdccb..4c7667a7c2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -25,7 +24,6 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CustomerId; @@ -44,7 +42,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -54,7 +51,6 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -421,22 +417,12 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); + Thread.sleep(3000); + MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); - client.disconnect(); - } - - private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { - long start = System.currentTimeMillis(); - while (!client.isConnected()) { - Thread.sleep(100); - if (start + ms < System.currentTimeMillis()) { - throw new RuntimeException("Client is not connected!"); - } - } } private Set getTelemetryKeys(String type, String id) throws Exception { @@ -463,13 +449,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); + Thread.sleep(3000); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - client.disconnect(); + return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 347eaee7eb..8dc0acff57 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.controller; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,9 +33,4 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index a9bb113a60..c43637e05f 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,13 +16,11 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -43,9 +41,4 @@ public class MqttNoSqlTestSuite { Arrays.asList( new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false)), "cassandra-test.yaml", 30000l); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 70de3b27ca..2863589ba1 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.mqtt; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -34,9 +32,4 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 9ddb2c81d9..5ba4dfcbae 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,7 +37,6 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -56,8 +55,6 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; - private static final AtomicInteger atomicInteger = new AtomicInteger(2); - @Before public void beforeTest() throws Exception { @@ -73,7 +70,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); + tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -112,8 +109,6 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); - Thread.sleep(2000); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -133,7 +128,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -142,7 +137,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -170,9 +165,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); - Thread.sleep(2000); - - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -190,7 +183,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -199,7 +192,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 47e9537ef9..f1580c65c7 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,8 +79,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(2000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + Thread.sleep(1000); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -97,25 +97,20 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - -// @Test - Unstable + @Test public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); + client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); - client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"uniqueValue\"}"; -// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. -// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) -// MqttClient <- SUB_ACK <- Transport - Thread.sleep(5000); - doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); + String payload = "{\"key\":\"value\"}"; + String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -125,8 +120,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private volatile Integer qoS; - private volatile String payload; + private Integer qoS; + private String payload; String getPayload() { return payload; @@ -143,7 +138,6 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { - log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 83ac5f7703..5f930821f7 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.rules; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,9 +33,4 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 97e6e59969..cfac1ea66b 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,21 +15,19 @@ */ package org.thingsboard.server.rules.flow; -import akka.actor.ActorRef; +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; @@ -39,14 +37,17 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.rule.RuleChainService; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -61,7 +62,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorSystemContext actorSystem; + protected ActorService actorService; @Autowired protected AttributesService attributesService; @@ -148,12 +149,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -260,13 +264,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - - Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 088f9390ff..1d35168e78 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,17 +15,15 @@ */ package org.thingsboard.server.rules.lifecycle; -import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; -import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -40,13 +38,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -63,7 +61,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorSystemContext actorSystem; + protected ActorService actorService; @Autowired protected AttributesService attributesService; @@ -138,13 +136,16 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); - + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", + null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java new file mode 100644 index 0000000000..d717169048 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.datastax.driver.core.utils.UUIDs; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ConsistentClusterRoutingServiceTest { + + private ConsistentClusterRoutingService clusterRoutingService; + + private DiscoveryService discoveryService; + + private String hashFunctionName = "murmur3_128"; + private Integer virtualNodesSize = 1024*4; + private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); + + + @Before + public void setup() throws Exception { + discoveryService = mock(DiscoveryService.class); + clusterRoutingService = new ConsistentClusterRoutingService(); + ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); + ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); + when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); + List otherServers = new ArrayList<>(); + for (int i = 1; i < 30; i++) { + otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); + } + when(discoveryService.getOtherServers()).thenReturn(otherServers); + clusterRoutingService.init(); + } + + @Test + public void testDispersionOnMillionDevices() { + List devices = new ArrayList<>(); + for (int i = 0; i < 1000000; i++) { + devices.add(new DeviceId(UUIDs.timeBased())); + } + + testDevicesDispersion(devices); + } + + private void testDevicesDispersion(List devices) { + long start = System.currentTimeMillis(); + Map map = new HashMap<>(); + for (DeviceId deviceId : devices) { + ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); + map.put(address, map.getOrDefault(address, 0) + 1); + } + + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + long end = System.currentTimeMillis(); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); + + for (Map.Entry entry : data) { +// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); + } + + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java deleted file mode 100644 index 3f7619ed1e..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.datastax.driver.core.utils.UUIDs; -import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class ConsistentHashParitionServiceTest { - - public static final int ITERATIONS = 1000000; - private ConsistentHashPartitionService clusterRoutingService; - - private TbServiceInfoProvider discoveryService; - private TenantRoutingInfoService routingInfoService; - private ApplicationEventPublisher applicationEventPublisher; - - private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 16; - - - @Before - public void setup() throws Exception { - discoveryService = mock(TbServiceInfoProvider.class); - applicationEventPublisher = mock(ApplicationEventPublisher.class); - routingInfoService = mock(TenantRoutingInfoService.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); - ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); - ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); - - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() - .setServiceId("100.96.1.1") - .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) - .build(); -// when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); - for (int i = 1; i < 30; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() - .setServiceId("100.96." + i * 2 + "." + i) - .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) - .build()); - } - clusterRoutingService.init(); - clusterRoutingService.recalculatePartitions(currentServer, otherServers); - } - - @Test - public void testDispersionOnMillionDevices() { - List devices = new ArrayList<>(); - for (int i = 0; i < ITERATIONS; i++) { - devices.add(new DeviceId(UUIDs.timeBased())); - } - testDevicesDispersion(devices); - } - - private void testDevicesDispersion(List devices) { - long start = System.currentTimeMillis(); - Map map = new HashMap<>(); - for (DeviceId deviceId : devices) { - TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); - Integer partition = address.getPartition().get(); - map.put(partition, map.getOrDefault(partition, 0) + 1); - } - - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); - long end = System.currentTimeMillis(); - double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); - - for (Map.Entry entry : data) { - System.out.println(entry.getKey() + ": " + entry.getValue()); - } - - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java deleted file mode 100644 index 43cd62ea97..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.runners.MockitoJUnitRunner; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class TbMsgPackProcessingContextTest { - - @Test - public void testHighConcurrencyCase() throws InterruptedException { - TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); - int msgCount = 1000; - int parallelCount = 5; - ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); - try { - ConcurrentMap> messages = new ConcurrentHashMap<>(); - for (int i = 0; i < msgCount; i++) { - messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); - } - when(strategyMock.getPendingMap()).thenReturn(messages); - TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); - for (UUID uuid : messages.keySet()) { - for (int i = 0; i < parallelCount; i++) { - executorService.submit(() -> context.onSuccess(uuid)); - } - } - Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); - Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); - } finally { - executorService.shutdownNow(); - } - } -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 62881bd5af..82b3df9736 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,7 +24,6 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -63,7 +62,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -79,7 +78,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -95,7 +94,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -113,7 +112,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -127,7 +126,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -148,7 +147,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -170,7 +169,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 8a89de5dff..1065fbf6d3 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,9 +49,4 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } - - @Override - protected long getMaxBlacklistDuration() { - return 100000; - } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java deleted file mode 100644 index c4182db3ee..0000000000 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.system; - -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.dao.CustomCassandraCQLUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; - -import java.util.Arrays; - -/** - * @author Andrew Shvayka - */ -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.system.*NoSqlTest"}) -public class SystemNoSqlTestSuite { - - @ClassRule - public static CustomCassandraCQLUnit cassandraUnit = - new CustomCassandraCQLUnit( - Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), - new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), - "cassandra-test.yaml", 30000l); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } -} diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index 6060a2cc2b..b12d513ce0 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.system; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,9 +35,4 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } - } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 298800a2f0..bf5a494afb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index a41dd2ee21..96a36e1f65 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index bcf1222988..20954b8ac2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index 7b10e0cf03..ebd0ef7d28 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index a888926bd0..ef73ec8f90 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index a215a16b29..f2a8800032 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 76db7920a6..c3d607f8fe 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.dao.util; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -@Retention(RetentionPolicy.RUNTIME) public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9be3321988..9a43c530a2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index 0b665c5695..abba0e985b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 0541a47879..20745d790c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index 766d405e25..b1f6a967b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,8 +29,6 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; - private boolean isolatedTbCore; - private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -74,22 +72,6 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } - public boolean isIsolatedTbCore() { - return isolatedTbCore; - } - - public void setIsolatedTbCore(boolean isolatedTbCore) { - this.isolatedTbCore = isolatedTbCore; - } - - public boolean isIsolatedTbRuleEngine() { - return isolatedTbRuleEngine; - } - - public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { - this.isolatedTbRuleEngine = isolatedTbRuleEngine; - } - @Override public String getSearchText() { return getTitle(); @@ -102,10 +84,6 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); - builder.append(", isolatedTbCore="); - builder.append(isolatedTbCore); - builder.append(", isolatedTbRuleEngine="); - builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index 9c23a60c28..a43d95975e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,7 +23,6 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java index 691d89cafd..532842ae75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.id; +package org.thingsboard.server.common.data.alarm; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 11db1da1a0..c367355cb0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 2993d81154..5639f98d01 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,10 +32,6 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } - public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { - this(kv, lastUpdateTs); - } - @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 0345355aa3..04b0cae56b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,9 +15,6 @@ */ package org.thingsboard.server.common.msg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; - /** * Created by ashvayka on 15.03.18. */ @@ -27,12 +24,17 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link PartitionChangeMsg} + * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} */ - PARTITION_CHANGE_MSG, + CLUSTER_EVENT_MSG, APP_INIT_MSG, + /** + * All messages, could be send to cluster + */ + SEND_TO_CLUSTER_MSG, + /** * ADDED/UPDATED/DELETED events for main entities. * @@ -41,11 +43,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. + * Misc messages from the REST API/SERVICE layer to the new rule engine. * - * See {@link QueueToRuleEngineMsg} + * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} */ - QUEUE_TO_RULE_ENGINE_MSG, + SERVICE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. @@ -89,9 +91,12 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, + DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, + /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ + DEVICE_ACTOR_TO_RULE_ENGINE_MSG, SESSION_TIMEOUT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 3edf4e5061..4e7090f934 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -15,27 +15,24 @@ */ package org.thingsboard.server.common.msg; -import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.UUID; /** * Created by ashvayka on 13.01.18. */ @Data -@Builder -@Slf4j +@AllArgsConstructor public final class TbMsg implements Serializable { private final UUID id; @@ -44,61 +41,30 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; + private final TbMsgTransactionData transactionData; + + //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - //This field is not serialized because we use queues and there is no need to do it - private final TbMsgCallback callback; - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); - } - - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); - } - - public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } + private final long clusterPartition; - private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; - this.dataType = dataType; this.data = data; + this.dataType = TbMsgDataType.JSON; + this.transactionData = new TbMsgTransactionData(id, originator); this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; - if (callback != null) { - this.callback = callback; - } else { - log.warn("[{}] Created message with empty callback: {}", originator, type); - this.callback = TbMsgCallback.EMPTY; - } + this.clusterPartition = clusterPartition; } - public static ByteString toByteString(TbMsg msg) { - return ByteString.copyFrom(toByteArray(msg)); + public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); } public static byte[] toByteArray(TbMsg msg) { @@ -123,37 +89,52 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } + TbMsgTransactionData transactionData = msg.getTransactionData(); + if (transactionData != null) { + MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); + transactionBuilder.setId(transactionData.getTransactionId().toString()); + transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); + transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); + transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); + builder.setTransactionData(transactionBuilder.build()); + } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); + } - public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { + public static ByteBuffer toBytes(TbMsg msg) { + return ByteBuffer.wrap(toByteArray(msg)); + } + + public static TbMsg fromBytes(byte[] data) { + return fromBytes(ByteBuffer.wrap(data)); + } + + public static TbMsg fromBytes(ByteBuffer buffer) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); + EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), + new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); + TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = null; + RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); RuleNodeId ruleNodeId = null; - if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { - ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); - } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); + public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); } - public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); - } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java new file mode 100644 index 0000000000..5d57514275 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +public final class TbMsgTransactionData implements Serializable { + + private final UUID transactionId; + private final EntityId originatorId; + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java similarity index 71% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java index 0e8d66aa63..56f928a1c8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java @@ -13,28 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.common.msg.cluster; import lombok.Data; -import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import java.util.Set; - /** * @author Andrew Shvayka */ @Data -public final class PartitionChangeMsg implements TbActorMsg { +public final class ClusterEventMsg implements TbActorMsg { - @Getter - private final ServiceQueueKey serviceQueueKey; - @Getter - private final Set partitions; + private final ServerAddress serverAddress; + private final boolean added; @Override public MsgType getMsgType() { - return MsgType.PARTITION_CHANGE_MSG; + return MsgType.CLUSTER_EVENT_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java index b258c6db1b..3326c33aa3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.common.msg.cluster; +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; -import java.util.UUID; +@Data +public class SendToClusterMsg implements TbActorMsg { -public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + private TbActorMsg msg; + private EntityId entityId; - public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); + public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { + this.entityId = entityId; + this.msg = msg; } - @Override - protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { - return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + @Override + public MsgType getMsgType() { + return MsgType.SEND_TO_CLUSTER_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java similarity index 50% rename from application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java index 5a5c172ee6..80674f9bce 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java @@ -13,32 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.msg.cluster; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TbCallback; +import lombok.Data; +import lombok.EqualsAndHashCode; -import java.util.UUID; +import java.io.Serializable; -@Slf4j -public class TbPackCallback implements TbCallback { - private final TbPackProcessingContext ctx; - private final UUID id; +/** + * @author Andrew Shvayka + */ +@Data +@EqualsAndHashCode +public class ServerAddress implements Comparable, Serializable { - public TbPackCallback(UUID id, TbPackProcessingContext ctx) { - this.id = id; - this.ctx = ctx; - } + private final String host; + private final int port; + private final ServerType serverType; @Override - public void onSuccess() { - log.trace("[{}] ON SUCCESS", id); - ctx.onSuccess(id); + public int compareTo(ServerAddress o) { + int result = this.host.compareTo(o.host); + if (result == 0) { + result = this.port - o.port; + } + return result; } @Override - public void onFailure(Throwable t) { - log.trace("[{}] ON FAILURE", id, t); - ctx.onFailure(id, t); + public String toString() { + return '[' + host + ':' + port + ']'; } } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java index d7a4715460..4b00284c34 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.subscription; +package org.thingsboard.server.common.msg.cluster; -public enum TbSubscriptionType { - TIMESERIES, ATTRIBUTES +/** + * Created by ashvayka on 23.09.18. + */ +public enum ServerType { + CORE } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java b/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java similarity index 76% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java index 2c5da0d3ea..d06046fd71 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java @@ -13,14 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.service; +package org.thingsboard.server.common.msg.core; import lombok.Data; -import java.util.UUID; - +/** + * @author Andrew Shvayka + */ @Data -public class RpcRequestMetadata { - private final UUID sessionId; +public class ToServerRpcResponseMsg { + private final int requestId; + private final String data; + + public boolean isSuccess() { + return true; + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index d05948206c..a52c237f87 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java deleted file mode 100644 index dd720251bb..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RuleEngineException extends Exception { - protected static final ObjectMapper mapper = new ObjectMapper(); - - public RuleEngineException(String message) { - super(message != null ? message : "Unknown"); - } - - public String toJsonString() { - try { - return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); - } catch (JsonProcessingException e) { - log.warn("Failed to serialize exception ", e); - throw new RuntimeException(e); - } - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java deleted file mode 100644 index 3f437dd1d7..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.data.rule.RuleNode; - -@Slf4j -public class RuleNodeException extends RuleEngineException { - @Getter - private final String ruleChainName; - @Getter - private final String ruleNodeName; - @Getter - private final RuleChainId ruleChainId; - @Getter - private final RuleNodeId ruleNodeId; - - public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { - super(message); - this.ruleChainName = ruleChainName; - this.ruleNodeName = ruleNode.getName(); - this.ruleChainId = ruleNode.getRuleChainId(); - this.ruleNodeId = ruleNode.getId(); - } - - public String toJsonString() { - try { - return mapper.writeValueAsString(mapper.createObjectNode() - .put("ruleNodeId", ruleNodeId.toString()) - .put("ruleChainId", ruleChainId.toString()) - .put("ruleNodeName", ruleNodeName) - .put("ruleChainName", ruleChainName) - .put("message", getMessage())); - } catch (JsonProcessingException e) { - log.warn("Failed to serialize exception ", e); - throw new RuntimeException(e); - } - } - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java deleted file mode 100644 index be08f054b6..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import lombok.ToString; - -import java.util.Objects; - -@ToString -public class ServiceQueue { - - public static final String MAIN = "Main"; - - private final ServiceType type; - private final String queue; - - public ServiceQueue(ServiceType type) { - this.type = type; - this.queue = MAIN; - } - - public ServiceQueue(ServiceType type, String queue) { - this.type = type; - this.queue = queue; - } - - public ServiceType getType() { - return type; - } - - public String getQueue() { - return queue; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceQueue that = (ServiceQueue) o; - return type == that.type && - queue.equals(that.queue); - } - - @Override - public int hashCode() { - return Objects.hash(type, queue); - } - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java deleted file mode 100644 index 0ca776126d..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -public enum ServiceType { - - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; - - public static ServiceType of(String serviceType) { - return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java deleted file mode 100644 index 9e8d5ae6b8..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -public interface TbMsgCallback { - - TbMsgCallback EMPTY = new TbMsgCallback() { - - @Override - public void onSuccess() { - - } - - @Override - public void onFailure(RuleEngineException e) { - - } - }; - - void onSuccess(); - - void onFailure(RuleEngineException e); - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java deleted file mode 100644 index bf0fffe404..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import lombok.Builder; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Objects; -import java.util.Optional; - -@ToString -public class TopicPartitionInfo { - - private final String topic; - private final TenantId tenantId; - private final Integer partition; - @Getter - private final String fullTopicName; - @Getter - private final boolean myPartition; - - @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { - this.topic = topic; - this.tenantId = tenantId; - this.partition = partition; - this.myPartition = myPartition; - String tmp = topic; - if (tenantId != null) { - tmp += "." + tenantId.getId().toString(); - } - if (partition != null) { - tmp += "." + partition; - } - this.fullTopicName = tmp; - } - - public String getTopic() { - return topic; - } - - public Optional getTenantId() { - return Optional.ofNullable(tenantId); - } - - public Optional getPartition() { - return Optional.ofNullable(partition); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TopicPartitionInfo that = (TopicPartitionInfo) o; - return topic.equals(that.topic) && - Objects.equals(tenantId, that.tenantId) && - Objects.equals(partition, that.partition) && - fullTopicName.equals(that.fullTopicName); - } - - @Override - public int hashCode() { - return Objects.hash(fullTopicName); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java similarity index 73% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java index fca1c51995..a717af85a4 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.common.msg.system; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -22,25 +22,18 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; -import java.util.Set; /** * Created by ashvayka on 15.03.18. */ @Data -public final class QueueToRuleEngineMsg implements TbActorMsg { +public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { private final TenantId tenantId; private final TbMsg tbMsg; - private final Set relationTypes; - private final String failureMessage; @Override public MsgType getMsgType() { - return MsgType.QUEUE_TO_RULE_ENGINE_MSG; - } - - public boolean isTellNext() { - return relationTypes != null && !relationTypes.isEmpty(); + return MsgType.SERVICE_TO_RULE_ENGINE_MSG; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java new file mode 100644 index 0000000000..5d47425e43 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.timeout; + +import org.thingsboard.server.common.msg.MsgType; + +/** + * @author Andrew Shvayka + */ +public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { + + public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { + super(id, timeout); + } + + @Override + public MsgType getMsgType() { + return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; + } +} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 737d5f6f57..03dc17b5c9 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,6 +23,13 @@ message TbMsgMetaDataProto { map data = 1; } +message TbMsgTransactionDataProto { + string id = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + message TbMsgProto { string id = 1; string type = 2; @@ -39,7 +46,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - //Transaction Data (12) was removed in 2.5 + TbMsgTransactionDataProto transactionData = 12; int32 dataType = 13; string data = 14; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index b4a223cac0..24881a0e3f 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,10 +40,6 @@ org.thingsboard.common data - - org.thingsboard.common - util - org.thingsboard.common message @@ -52,22 +48,6 @@ org.apache.kafka kafka-clients - - com.amazonaws - aws-java-sdk-sqs - - - com.google.cloud - google-cloud-pubsub - - - com.microsoft.azure - azure-servicebus - - - com.rabbitmq - amqp-client - org.springframework spring-context-support @@ -104,14 +84,6 @@ ch.qos.logback logback-classic - - com.google.protobuf - protobuf-java - - - org.apache.curator - curator-recipes - junit junit @@ -124,13 +96,4 @@ - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java index ec3b0b537a..0c68c8dd5d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java @@ -13,17 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; + +import lombok.extern.slf4j.Slf4j; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; -public class AbstractTbQueueTemplate { - protected static final String REQUEST_ID_HEADER = "requestId"; - protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - protected static final String REQUEST_TIME = "requestTime"; - +/** + * Created by ashvayka on 25.09.18. + */ +@Slf4j +public abstract class AbstractTbKafkaTemplate { protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); @@ -45,14 +47,4 @@ public class AbstractTbQueueTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - - protected static byte[] longToBytes(long x) { - ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); - longBuffer.putLong(0, x); - return longBuffer.array(); - } - - protected static long bytesToLong(byte[] bytes) { - return ByteBuffer.wrap(bytes).getLong(); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java index aa87df031a..a01411e48f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java new file mode 100644 index 0000000000..0fe86e030b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import org.apache.kafka.clients.admin.*; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.KafkaFuture; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Created by ashvayka on 24.09.18. + */ +public class TBKafkaAdmin { + + AdminClient client; + + public TBKafkaAdmin(TbKafkaSettings settings) { + client = AdminClient.create(settings.toProps()); + } + + public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { + synchronized (this) { + long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); + while (!topicExists(topic)) { + long waitMs = timeoutExpiredMs - System.currentTimeMillis(); + if (waitMs <= 0) { + throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); + } else { + wait(1000); + } + } + } + } + + public CreateTopicsResult createTopic(NewTopic topic){ + return client.createTopics(Collections.singletonList(topic)); + } + + private boolean topicExists(String topic) throws InterruptedException { + KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); + try { + topicDescriptionFuture.get(); + return true; + } catch (ExecutionException e) { + return false; + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java new file mode 100644 index 0000000000..73f8cc16c7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.Getter; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.UUID; + +/** + * Created by ashvayka on 24.09.18. + */ +public class TBKafkaConsumerTemplate { + + private final KafkaConsumer consumer; + private final TbKafkaDecoder decoder; + + @Builder.Default + private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); + + @Getter + private final String topic; + + @Builder + private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + TbKafkaRequestIdExtractor requestIdExtractor, + String clientId, String groupId, String topic, + boolean autoCommit, int autoCommitIntervalMs, + int maxPollRecords) { + Properties props = settings.toProps(); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + if (groupId != null) { + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + } + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + if (maxPollRecords > 0) { + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + } + this.consumer = new KafkaConsumer<>(props); + this.decoder = decoder; + this.requestIdExtractor = requestIdExtractor; + this.topic = topic; + } + + public void subscribe() { + consumer.subscribe(Collections.singletonList(topic)); + } + + public void unsubscribe() { + consumer.unsubscribe(); + } + + public ConsumerRecords poll(Duration duration) { + return consumer.poll(duration); + } + + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(record.value()); + } + + public UUID extractRequestId(T value) { + return requestIdExtractor.extractRequestId(value); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java new file mode 100644 index 0000000000..8722c1d64e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.header.Header; +import org.springframework.util.StringUtils; + +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TBKafkaProducerTemplate { + + private final KafkaProducer producer; + private final TbKafkaEncoder encoder; + + private final TbKafkaPartitioner partitioner; + private ConcurrentMap> partitionInfoMap; + @Getter + private final String defaultTopic; + + @Getter + private final TbKafkaSettings settings; + + @Builder + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, + TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + Properties props = settings.toProps(); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + if (!StringUtils.isEmpty(clientId)) { + props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + } + this.settings = settings; + this.producer = new KafkaProducer<>(props); + this.encoder = encoder; + this.partitioner = partitioner; + this.defaultTopic = defaultTopic; + } + + public void init() { + this.partitionInfoMap = new ConcurrentHashMap<>(); + if (!StringUtils.isEmpty(defaultTopic)) { + try { + TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); + admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); + log.info("[{}] Topic exists.", defaultTopic); + } catch (Exception e) { + log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); + throw new RuntimeException(e); + } + //Maybe this should not be cached, but we don't plan to change size of partitions + this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); + } + } + + public Future send(String key, T value, Callback callback) { + return send(key, value, null, callback); + } + + public Future send(String key, T value, Iterable
headers, Callback callback) { + return send(key, value, null, headers, callback); + } + + public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { + if (!StringUtils.isEmpty(this.defaultTopic)) { + return send(this.defaultTopic, key, value, timestamp, headers, callback); + } else { + throw new RuntimeException("Failed to send message! Default topic is not specified!"); + } + } + + public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { + return send(topic, key, value, null, headers, callback); + } + + public Future send(String topic, String key, T value, Callback callback) { + return send(topic, key, value, null, null, callback); + } + + public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { + byte[] data = encoder.encode(value); + ProducerRecord record; + Integer partition = getPartition(topic, key, value, data); + record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); + return producer.send(record, callback); + } + + private Integer getPartition(String topic, String key, T value, byte[] data) { + if (partitioner == null) { + return null; + } else { + return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java index 6e3ea1e4a9..ab196d5863 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java @@ -13,9 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; - -import org.thingsboard.server.queue.TbQueueMsg; +package org.thingsboard.server.kafka; import java.io.IOException; @@ -24,6 +22,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(TbQueueMsg msg) throws IOException; + T decode(byte[] data) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java index b3c4dec8ee..22f65c58a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java index 609cbd2107..5e5aa0591a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.ListenableFuture; /** * Created by ashvayka on 05.10.18. */ -public interface TbQueueHandler { +public interface TbKafkaHandler { ListenableFuture handle(Request request); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java index 1b2f984093..8ec5cd1d97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java @@ -13,25 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.apache.kafka.clients.producer.Partitioner; +import org.apache.kafka.common.PartitionInfo; import java.util.List; -import java.util.Set; -public interface TbQueueConsumer { - - String getTopic(); - - void subscribe(); - - void subscribe(Set partitions); - - void unsubscribe(); - - List poll(long durationInMillis); +/** + * Created by ashvayka on 25.09.18. + */ +public interface TbKafkaPartitioner extends Partitioner { - void commit(); + int partition(String topic, String key, T value, byte[] encodedValue, List partitions); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java index b6de7db50d..cd52948da8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java @@ -13,9 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java index b0e4990305..6e433e71a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java @@ -13,15 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; import java.util.UUID; -public interface TbQueueMsg { +public interface TbKafkaRequestIdExtractor { - UUID getKey(); + UUID extractRequestId(T value); - TbQueueMsgHeaders getHeaders(); - - byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java index f02ae63441..2a7bca119f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java @@ -13,22 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -37,14 +41,15 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; +/** + * Created by ashvayka on 25.09.18. + */ @Slf4j -public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate - implements TbQueueRequestTemplate { +public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { - private final TbQueueAdmin queueAdmin; - private final TbQueueProducer requestTemplate; - private final TbQueueConsumer responseTemplate; - private final ConcurrentMap> pendingRequests; + private final TBKafkaProducerTemplate requestTemplate; + private final TBKafkaConsumerTemplate responseTemplate; + private final ConcurrentMap> pendingRequests; private final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; @@ -55,14 +60,12 @@ public class DefaultTbQueueRequestTemplate requestTemplate, - TbQueueConsumer responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { - this.queueAdmin = queueAdmin; + public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, + TBKafkaConsumerTemplate responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { this.requestTemplate = requestTemplate; this.responseTemplate = responseTemplate; this.pendingRequests = new ConcurrentHashMap<>(); @@ -78,9 +81,20 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); - if (responses.size() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.size()); - } else { - continue; + ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); + if (responses.count() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.count()); } responses.forEach(response -> { - byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId; + log.trace("Received response to Kafka Template request: {}", response); + Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); + Response decodedResponse = null; + UUID requestId = null; if (requestIdHeader == null) { + try { + decodedResponse = responseTemplate.decode(response); + requestId = responseTemplate.extractRequestId(decodedResponse); + } catch (IOException e) { + log.error("Failed to decode response", e); + } + } else { + requestId = bytesToUuid(requestIdHeader.value()); + } + if (requestId == null) { log.error("[{}] Missing requestId in header and body", response); } else { - requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received: {}", requestId, response); + log.trace("[{}] Response received", requestId); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); } else { - expectedResponse.future.set(response); + try { + if (decodedResponse == null) { + decodedResponse = responseTemplate.decode(response); + } + expectedResponse.future.set(decodedResponse); + } catch (IOException e) { + expectedResponse.future.setException(e); + } } } }); - responseTemplate.commit(); tickTs = System.currentTimeMillis(); tickSize = pendingRequests.size(); if (nextCleanupMs < tickTs) { @@ -126,6 +155,10 @@ public class DefaultTbQueueRequestTemplate send(Request request) { + public ListenableFuture post(String key, Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); - request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); - request.getHeaders().put(REQUEST_TIME, longToBytes(System.currentTimeMillis())); + List
headers = new ArrayList<>(2); + headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); + headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); SettableFuture future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - log.trace("[{}] Request sent: {}", requestId, metadata); - } - - @Override - public void onFailure(Throwable t) { - pendingRequests.remove(requestId); - future.setException(t); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + requestTemplate.send(key, request, headers, (metadata, exception) -> { + if (exception != null) { + log.trace("[{}] Failed to post the request", requestId, exception); + } else { + log.trace("[{}] Posted the request: {}", requestId, metadata); } }); return future; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java new file mode 100644 index 0000000000..864182a552 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; + +import java.time.Duration; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by ashvayka on 25.09.18. + */ +@Slf4j +public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { + + private final TBKafkaConsumerTemplate requestTemplate; + private final TBKafkaProducerTemplate responseTemplate; + private final TbKafkaHandler handler; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, + TBKafkaProducerTemplate responseTemplate, + TbKafkaHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.handler = handler; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + public void init() { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); + requests.forEach(request -> { + Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader.value()); + if (requestId == null) { + log.error("[{}] Missing requestId in header and body", request); + return; + } + Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + String responseTopic = bytesToString(responseTopicHeader.value()); + try { + pendingRequestCount.getAndIncrement(); + Request decodedRequest = requestTemplate.decode(request); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), + response -> { + pendingRequestCount.decrementAndGet(); + reply(requestId, responseTopic, response); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timedout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + }); + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from kafka was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + + private void reply(UUID requestId, String topic, Response response) { + responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 66121cb215..6446643a18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import java.util.List; @@ -29,32 +28,31 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) @Component public class TbKafkaSettings { - @Value("${queue.kafka.bootstrap.servers}") + static final String REQUEST_ID_HEADER = "requestId"; + static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + @Value("${kafka.bootstrap.servers}") private String servers; - @Value("${queue.kafka.acks}") + @Value("${kafka.acks}") private String acks; - @Value("${queue.kafka.retries}") + @Value("${kafka.retries}") private int retries; - @Value("${queue.kafka.batch.size}") + @Value("${kafka.batch.size}") private int batchSize; - @Value("${queue.kafka.linger.ms}") + @Value("${kafka.linger.ms}") private long lingerMs; - @Value("${queue.kafka.buffer.memory}") + @Value("${kafka.buffer.memory}") private long bufferMemory; - @Value("${queue.kafka.replication_factor}") - @Getter - private short replicationFactor; - @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java similarity index 50% rename from common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java index 922a2b1062..343c5e6dd8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java @@ -13,30 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.sqs; +package org.thingsboard.server.kafka; -import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Created by ashvayka on 12.10.18. + */ @Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") @Component -@Data -public class TbAwsSqsSettings { - - @Value("${queue.aws_sqs.access_key_id}") - private String accessKeyId; - - @Value("${queue.aws_sqs.secret_access_key}") - private String secretAccessKey; +public class TbNodeIdProvider { - @Value("${queue.aws_sqs.region}") - private String region; + @Getter + @Value("${cluster.node_id:#{null}}") + private String nodeId; - @Value("${queue.aws_sqs.threads_per_topic}") - private int threadsPerTopic; + @PostConstruct + public void init() { + if (StringUtils.isEmpty(nodeId)) { + try { + nodeId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current NodeId: {}", nodeId); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java deleted file mode 100644 index f21ad80d10..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueCallback { - - void onSuccess(TbQueueMsgMetadata metadata); - - void onFailure(Throwable t); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java deleted file mode 100644 index 4cb38f6685..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.thingsboard.server.queue.TbQueueMsg; - -public interface TbQueueMsgDecoder { - - T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java deleted file mode 100644 index f205ed676f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import java.util.Map; - -public interface TbQueueMsgHeaders { - - byte[] put(String key, byte[] value); - - byte[] get(String key); - - Map getData(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java deleted file mode 100644 index 0c24fec438..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueMsgMetadata { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java deleted file mode 100644 index ef308291a9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -public interface TbQueueProducer { - - void init(); - - String getDefaultTopic(); - - void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); - - void stop(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java deleted file mode 100644 index 20530360d6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import com.google.common.util.concurrent.ListenableFuture; - -public interface TbQueueRequestTemplate { - - void init(); - - ListenableFuture send(Request request); - - void stop(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java deleted file mode 100644 index c823b3df88..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueResponseTemplate { - - void init(TbQueueHandler handler); - - void stop(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java deleted file mode 100644 index 336514b7b2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.microsoft.azure.servicebus.management.ManagementClient; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -public class TbServiceBusAdmin implements TbQueueAdmin { - - private final Set queues = ConcurrentHashMap.newKeySet(); - - private final ManagementClient client; - - public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { - ConnectionStringBuilder builder = new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - "queues", - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - - client = new ManagementClient(builder); - - try { - client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to get queues.", e); - throw new RuntimeException("Failed to get queues.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic) { - if (queues.contains(topic)) { - return; - } - - try { - client.createQueue(topic); - queues.add(topic); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to create queue: [{}]", topic, e); - } - } - - public void destroy() { - try { - client.close(); - } catch (IOException e) { - log.error("Failed to close ManagementClient."); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java deleted file mode 100644 index cca599d59a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.microsoft.azure.servicebus.TransactionContext; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; -import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; -import com.microsoft.azure.servicebus.primitives.MessagingEntityType; -import com.microsoft.azure.servicebus.primitives.MessagingFactory; -import com.microsoft.azure.servicebus.primitives.SettleModePair; -import lombok.extern.slf4j.Slf4j; -import org.apache.qpid.proton.amqp.messaging.Data; -import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; -import org.apache.qpid.proton.amqp.transport.SenderSettleMode; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbServiceBusConsumerTemplate implements TbQueueConsumer { - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbServiceBusSettings serviceBusSettings; - - private final Gson gson = new Gson(); - - private Set receivers; - private volatile Set partitions; - private volatile boolean subscribed; - private volatile boolean stopped = false; - private Map> pendingMessages = new ConcurrentHashMap<>(); - private volatile int messagesPerQueue; - - public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.serviceBusSettings = serviceBusSettings; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - receivers.forEach(CoreMessageReceiver::closeAsync); - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - createReceivers(); - messagesPerQueue = receivers.size() / partitions.size(); - subscribed = true; - } - - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to parse message.", e); - throw new RuntimeException("Failed to parse message.", e); - } - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - private void createReceivers() { - List> receiverFutures = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .map(queue -> { - MessagingFactory factory; - try { - factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to create factory for the queue [{}]", queue); - throw new RuntimeException("Failed to create the factory", e); - } - - return CoreMessageReceiver.create(factory, queue, queue, 0, - new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), - MessagingEntityType.QUEUE); - }).collect(Collectors.toList()); - - try { - receivers = new HashSet<>(fromList(receiverFutures).get()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to create receivers", e); - } - } - } - - private ConnectionStringBuilder createConnection(String queue) { - admin.createTopicIfNotExists(queue); - return new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - queue, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - } - - private CompletableFuture> fromList(List> futures) { - CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; - futures.toArray(arrayFuture); - - return CompletableFuture - .allOf(arrayFuture) - .thenApply(v -> futures - .stream() - .map(CompletableFuture::join) - .collect(Collectors.toList())); - } - - @Override - public void commit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java deleted file mode 100644 index 5d9a931378..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.microsoft.azure.servicebus.IMessage; -import com.microsoft.azure.servicebus.Message; -import com.microsoft.azure.servicebus.QueueClient; -import com.microsoft.azure.servicebus.ReceiveMode; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -@Slf4j -public class TbServiceBusProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbServiceBusSettings serviceBusSettings; - private final Map clients = new HashMap<>(); - private ExecutorService executorService; - - public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.serviceBusSettings = serviceBusSettings; - executorService = Executors.newSingleThreadExecutor(); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); - CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); - future.whenCompleteAsync((success, err) -> { - if (err != null) { - callback.onFailure(err); - } else { - callback.onSuccess(null); - } - }, executorService); - } - - @Override - public void stop() { - clients.forEach((t, client) -> { - try { - client.close(); - } catch (ServiceBusException e) { - log.error("Failed to close QueueClient.", e); - } - }); - - if (executorService != null) { - executorService.shutdownNow(); - } - } - - private QueueClient getClient(String topic) { - return clients.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - ConnectionStringBuilder builder = - new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - topic, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - try { - return new QueueClient(builder, ReceiveMode.PEEKLOCK); - } catch (InterruptedException | ServiceBusException e) { - log.error("Failed to create new client for the Queue: [{}]", topic, e); - throw new RuntimeException("Failed to create new client for the Queue", e); - } - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java deleted file mode 100644 index d872dcec0b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -@Component -@Data -public class TbServiceBusSettings { - @Value("${queue.service_bus.namespace_name}") - private String namespaceName; - @Value("${queue.service_bus.sas_key_name}") - private String sasKeyName; - @Value("${queue.service_bus.sas_key}") - private String sasKey; - @Value("${queue.service_bus.max_messages}") - private int maxMessages; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java deleted file mode 100644 index 7584e8c2d0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.UUID; - -@Data -public class DefaultTbQueueMsg implements TbQueueMsg { - private final UUID key; - private final byte[] data; - private final DefaultTbQueueMsgHeaders headers; - - public DefaultTbQueueMsg(TbQueueMsg msg) { - this.key = msg.getKey(); - this.data = msg.getData(); - DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - msg.getHeaders().getData().forEach(headers::put); - this.headers = headers; - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java deleted file mode 100644 index 36c3dd0999..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.util.HashMap; -import java.util.Map; - -public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { - - protected final Map data = new HashMap<>(); - - @Override - public byte[] put(String key, byte[] value) { - return data.put(key, value); - } - - @Override - public byte[] get(String key) { - return data.get(key); - } - - @Override - public Map getData() { - return data; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java deleted file mode 100644 index 96891ee7d8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueHandler; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate - implements TbQueueResponseTemplate { - - private final TbQueueConsumer requestTemplate; - private final TbQueueProducer responseTemplate; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, - TbQueueProducer responseTemplate, - TbQueueHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - @Override - public void init(TbQueueHandler handler) { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - List requests = requestTemplate.poll(pollInterval); - - if (requests.isEmpty()) { - continue; - } - - requests.forEach(request -> { - long currentTime = System.currentTimeMillis(); - long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); - if (requestTime + requestTimeout >= currentTime) { - byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader); - String responseTopic = bytesToString(responseTopicHeader); - try { - pendingRequestCount.getAndIncrement(); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), - response -> { - pendingRequestCount.decrementAndGet(); - response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timeout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - } - }); - requestTemplate.commit(); - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (requestTemplate != null) { - requestTemplate.unsubscribe(); - } - if (responseTemplate != null) { - responseTemplate.stop(); - } - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java deleted file mode 100644 index f860d8dcc9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -import java.util.concurrent.atomic.AtomicInteger; - -public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { - - private final AtomicInteger tbQueueCallbackCount; - private final TbMsgCallback tbMsgCallback; - - public MultipleTbQueueTbMsgCallbackWrapper(int tbQueueCallbackCount, TbMsgCallback tbMsgCallback) { - this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); - this.tbMsgCallback = tbMsgCallback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (tbQueueCallbackCount.decrementAndGet() <= 0) { - tbMsgCallback.onSuccess(); - } - } - - @Override - public void onFailure(Throwable t) { - tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java deleted file mode 100644 index 07417c4a9e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -public class TbProtoJsQueueMsg extends TbProtoQueueMsg { - - public TbProtoJsQueueMsg(UUID key, T value) { - super(key, value); - } - - public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { - super(key, value, headers); - } - - @Override - public byte[] getData() { - try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java deleted file mode 100644 index 2eb76950dc..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.util.UUID; - -@Data -public class TbProtoQueueMsg implements TbQueueMsg { - - private final UUID key; - protected final T value; - private final TbQueueMsgHeaders headers; - - public TbProtoQueueMsg(UUID key, T value) { - this(key, value, new DefaultTbQueueMsgHeaders()); - } - - public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { - this.key = key; - this.value = value; - this.headers = headers; - } - - @Override - public UUID getKey() { - return key; - } - - @Override - public TbQueueMsgHeaders getHeaders() { - return headers; - } - - @Override - public byte[] getData() { - return value.toByteArray(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java deleted file mode 100644 index 8bd9c3516b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { - - private final TbMsgCallback tbMsgCallback; - - public TbQueueTbMsgCallbackWrapper(TbMsgCallback tbMsgCallback) { - this.tbMsgCallback = tbMsgCallback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - tbMsgCallback.onSuccess(); - } - - @Override - public void onFailure(Throwable t) { - tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java deleted file mode 100644 index f0039d0f38..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; - -import java.util.Set; - - -public class ClusterTopologyChangeEvent extends ApplicationEvent { - - @Getter - private final Set serviceQueueKeys; - - public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { - super(source); - this.serviceQueueKeys = serviceQueueKeys; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java deleted file mode 100644 index df25ea3ba9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ /dev/null @@ -1,339 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -import org.thingsboard.server.common.msg.queue.ServiceQueue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; - -import javax.annotation.PostConstruct; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class ConsistentHashPartitionService implements PartitionService { - - @Value("${queue.core.topic}") - private String coreTopic; - @Value("${queue.core.partitions:100}") - private Integer corePartitions; - @Value("${queue.partitions.hash_function_name:murmur3_128}") - private String hashFunctionName; - @Value("${queue.partitions.virtual_nodes_size:16}") - private Integer virtualNodesSize; - - private final ApplicationEventPublisher applicationEventPublisher; - private final TbServiceInfoProvider serviceInfoProvider; - private final TenantRoutingInfoService tenantRoutingInfoService; - private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); - private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); - - private Map tbCoreNotificationTopics = new HashMap<>(); - private Map tbRuleEngineNotificationTopics = new HashMap<>(); - private List currentOtherServices; - - private HashFunction hashFunction; - - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { - this.serviceInfoProvider = serviceInfoProvider; - this.tenantRoutingInfoService = tenantRoutingInfoService; - this.applicationEventPublisher = applicationEventPublisher; - } - - @PostConstruct - public void init() { - this.hashFunction = forName(hashFunctionName); - partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); - partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - return resolve(new ServiceQueue(serviceType), tenantId, entityId); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { - return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); - } - - private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizes.get(serviceQueue); - int partition; - if (partitionSize != null) { - partition = Math.abs(hash % partitionSize); - } else { - //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. - partition = 0; - } - boolean isolatedTenant = isIsolated(serviceQueue, tenantId); - TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); - return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); - } - - @Override - public void recalculatePartitions(ServiceInfo currentService, List otherServices) { - logServiceInfo(currentService); - otherServices.forEach(this::logServiceInfo); - Map> circles = new HashMap<>(); - addNode(circles, currentService); - for (ServiceInfo other : otherServices) { - addNode(circles, other); - } - ConcurrentMap> oldPartitions = myPartitions; - TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); - myPartitions = new ConcurrentHashMap<>(); - partitionSizes.forEach((type, size) -> { - ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); - for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); - if (currentService.equals(serviceInfo)) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); - myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); - } - } - }); - myPartitions.forEach((serviceQueueKey, partitions) -> { - if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { - log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) - .collect(Collectors.toSet()); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); - } - }); - tpiCache.clear(); - - if (currentOtherServices == null) { - currentOtherServices = new ArrayList<>(otherServices); - } else { - Set changes = new HashSet<>(); - Map> currentMap = getServiceKeyListMap(currentOtherServices); - Map> newMap = getServiceKeyListMap(otherServices); - currentOtherServices = otherServices; - currentMap.forEach((key, list) -> { - if (!list.equals(newMap.get(key))) { - changes.add(key); - } - }); - currentMap.keySet().forEach(newMap::remove); - changes.addAll(newMap.keySet()); - if (!changes.isEmpty()) { - applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); - } - } - } - - @Override - public Set getAllServiceIds(ServiceType serviceType) { - Set result = new HashSet<>(); - ServiceInfo current = serviceInfoProvider.getServiceInfo(); - if (current.getServiceTypesList().contains(serviceType.name())) { - result.add(current.getServiceId()); - } - for (ServiceInfo serviceInfo : currentOtherServices) { - if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { - result.add(serviceInfo.getServiceId()); - } - } - return result; - } - - @Override - public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { - switch (serviceType) { - case TB_CORE: - return tbCoreNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); - case TB_RULE_ENGINE: - return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); - default: - return buildNotificationsTopicPartitionInfo(serviceType, serviceId); - } - } - - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); - services.forEach(serviceInfo -> { - for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); - } - } else { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); - } - } - }); - return currentMap; - } - - private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { - return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceQueue)); - tpi.partition(partition); - ServiceQueueKey myPartitionsSearchKey; - if (isIsolated(serviceQueue, tenantId)) { - tpi.tenantId(tenantId); - myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); - } else { - myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); - } - List partitions = myPartitions.get(myPartitionsSearchKey); - if (partitions != null) { - tpi.myPartition(partitions.contains(partition)); - } else { - tpi.myPartition(false); - } - return tpi.build(); - } - - private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - return false; - } - TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - synchronized (tenantRoutingInfoMap) { - routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); - tenantRoutingInfoMap.put(tenantId, routingInfo); - } - } - } - if (routingInfo == null) { - throw new RuntimeException("Tenant not found!"); - } - switch (serviceQueue.getType()) { - case TB_CORE: - return routingInfo.isIsolatedTbCore(); - case TB_RULE_ENGINE: - return routingInfo.isIsolatedTbRuleEngine(); - default: - return false; - } - } - - private void logServiceInfo(TransportProtos.ServiceInfo server) { - TenantId tenantId = getSystemOrIsolatedTenantId(server); - if (tenantId.isNullUid()) { - log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); - } else { - log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); - } - } - - private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { - return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); - } - - private void addNode(Map> circles, ServiceInfo instance) { - TenantId tenantId = getSystemOrIsolatedTenantId(instance); - for (String serviceTypeStr : instance.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); - partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); - partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); - } - } - } else { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); - } - } - } - } - - private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { - if (circle == null || circle.isEmpty()) { - return null; - } - Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - return circle.get(hash); - } - - private HashCode hash(ServiceInfo instance, int i) { - return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); - } - - public static HashFunction forName(String name) { - switch (name) { - case "murmur3_32": - return Hashing.murmur3_32(); - case "murmur3_128": - return Hashing.murmur3_128(); - case "crc32": - return Hashing.crc32(); - case "md5": - return Hashing.md5(); - default: - throw new IllegalArgumentException("Can't find hash function with name " + name); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java deleted file mode 100644 index 1c60a03cdd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; - -@Component -@Slf4j -public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { - - @Getter - @Value("${service.id:#{null}}") - private String serviceId; - - @Getter - @Value("${service.type:monolith}") - private String serviceType; - - @Getter - @Value("${service.tenant_id:}") - private String tenantIdStr; - - @Autowired(required = false) - private TbQueueRuleEngineSettings ruleEngineSettings; - - private List serviceTypes; - private ServiceInfo serviceInfo; - private TenantId isolatedTenant; - - @PostConstruct - public void init() { - if (StringUtils.isEmpty(serviceId)) { - try { - serviceId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current Service ID: {}", serviceId); - if (serviceType.equalsIgnoreCase("monolith")) { - serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); - } else { - serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); - } - ServiceInfo.Builder builder = ServiceInfo.newBuilder() - .setServiceId(serviceId) - .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); - UUID tenantId; - if (!StringUtils.isEmpty(tenantIdStr)) { - tenantId = UUID.fromString(tenantIdStr); - isolatedTenant = new TenantId(tenantId); - } else { - tenantId = TenantId.NULL_UUID; - } - builder.setTenantIdMSB(tenantId.getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); - - if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { - for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { - TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() - .setName(queue.getName()) - .setTopic(queue.getTopic()) - .setPartitions(queue.getPartitions()).build(); - builder.addRuleEngineQueues(queueInfo); - } - } - - serviceInfo = builder.build(); - } - - @Override - public ServiceInfo getServiceInfo() { - return serviceInfo; - } - - @Override - public boolean isService(ServiceType serviceType) { - return serviceTypes.contains(serviceType); - } - - @Override - public Optional getIsolatedTenant() { - return Optional.ofNullable(isolatedTenant); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java deleted file mode 100644 index 19b8f665a8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -import java.util.Set; - - -public class PartitionChangeEvent extends ApplicationEvent { - - @Getter - private final ServiceQueueKey serviceQueueKey; - @Getter - private final Set partitions; - - public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { - super(source); - this.serviceQueueKey = serviceQueueKey; - this.partitions = partitions; - } - - public ServiceType getServiceType() { - return serviceQueueKey.getServiceQueue().getType(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java deleted file mode 100644 index e3b3e76b80..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; -import java.util.Set; - -/** - * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} - */ -public interface PartitionService { - - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); - - TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); - - /** - * Received from the Discovery service when network topology is changed. - * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} - * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} - */ - void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); - - /** - * Get all active service ids by service type - * @param serviceType to filter the list of services - * @return list of all active services - */ - Set getAllServiceIds(ServiceType serviceType); - - /** - * Each Service should start a consumer for messages that target individual service instance based on serviceId. - * This topic is likely to have single partition, and is always assigned to the service. - * @param serviceType - * @param serviceId - * @return - */ - TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java deleted file mode 100644 index 07c4c2d9c5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Data; -import org.thingsboard.server.common.data.id.TenantId; - -@Data -public class TenantRoutingInfo { - private final TenantId tenantId; - private final boolean isolatedTbCore; - private final boolean isolatedTbRuleEngine; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java deleted file mode 100644 index 685a36df30..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import org.thingsboard.server.common.data.id.TenantId; - -public interface TenantRoutingInfoService { - - TenantRoutingInfo getRoutingInfo(TenantId tenantId); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java deleted file mode 100644 index 37661e85e4..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.AllArgsConstructor; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueue; - -import java.util.Objects; - -@AllArgsConstructor -public class TopicPartitionInfoKey { - private ServiceQueue serviceQueue; - private TenantId isolatedTenantId; - private int partition; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; - return partition == that.partition && - serviceQueue.equals(that.serviceQueue) && - Objects.equals(isolatedTenantId, that.isolatedTenantId); - } - - @Override - public int hashCode() { - return Objects.hash(serviceQueue, isolatedTenantId, partition); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java deleted file mode 100644 index 98470eafba..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; - -import java.util.UUID; - -public class KafkaTbQueueMsg implements TbQueueMsg { - private final UUID key; - private final TbQueueMsgHeaders headers; - private final byte[] data; - - public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - record.headers().forEach(header -> { - headers.put(header.key(), header.value()); - }); - this.headers = headers; - this.data = record.value(); - } - - @Override - public UUID getKey() { - return key; - } - - @Override - public TbQueueMsgHeaders getHeaders() { - return headers; - } - - @Override - public byte[] getData() { - return data; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java deleted file mode 100644 index 5a9eaa632c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -@Data -@AllArgsConstructor -public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { - private RecordMetadata metadata; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java deleted file mode 100644 index b92b094af1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.common.errors.TopicExistsException; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaAdmin implements TbQueueAdmin { - - private final AdminClient client; - private final Map topicConfigs; - private final Set topics = ConcurrentHashMap.newKeySet(); - - private final short replicationFactor; - - public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { - client = AdminClient.create(settings.toProps()); - this.topicConfigs = topicConfigs; - - try { - topics.addAll(client.listTopics().names().get()); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to get all topics.", e); - } - - replicationFactor = settings.getReplicationFactor(); - } - - @Override - public void createTopicIfNotExists(String topic) { - if (topics.contains(topic)) { - return; - } - try { - NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); - createTopic(newTopic).values().get(topic).get(); - topics.add(topic); - } catch (ExecutionException ee) { - if (ee.getCause() instanceof TopicExistsException) { - //do nothing - } else { - log.warn("[{}] Failed to create topic", topic, ee); - throw new RuntimeException(ee); - } - } catch (Exception e) { - log.warn("[{}] Failed to create topic", topic, e); - throw new RuntimeException(e); - } - - } - - @Override - public void destroy() { - if (client != null) { - client.close(); - } - } - - public CreateTopicsResult createTopic(NewTopic topic) { - return client.createTopics(Collections.singletonList(topic)); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java deleted file mode 100644 index 49d1d74102..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaConsumerTemplate implements TbQueueConsumer { - - private final TbQueueAdmin admin; - private final KafkaConsumer consumer; - private final TbKafkaDecoder decoder; - private volatile boolean subscribed; - private volatile Set partitions; - private final Lock consumerLock; - - @Getter - private final String topic; - - @Builder - private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - String clientId, String groupId, String topic, - boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords, - TbQueueAdmin admin) { - Properties props = settings.toProps(); - props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - if (groupId != null) { - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); - } - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (maxPollRecords > 0) { - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); - } - this.admin = admin; - this.consumer = new KafkaConsumer<>(props); - this.decoder = decoder; - this.topic = topic; - this.consumerLock = new ReentrantLock(); - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - try { - consumerLock.lock(); - - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; - } - } finally { - consumerLock.unlock(); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - consumerLock.lock(); - consumer.commitAsync(); - } finally { - consumerLock.unlock(); - } - } - - @Override - public void unsubscribe() { - try { - consumerLock.lock(); - if (consumer != null) { - consumer.unsubscribe(); - consumer.close(); - } - } finally { - consumerLock.unlock(); - } - } - - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(new KafkaTbQueueMsg(record)); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java deleted file mode 100644 index 4f26f51da7..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; - -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaProducerTemplate implements TbQueueProducer { - - private final KafkaProducer producer; - - @Getter - private final String defaultTopic; - - @Getter - private final TbKafkaSettings settings; - - private final TbQueueAdmin admin; - - private final Set topics; - - @Builder - private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { - Properties props = settings.toProps(); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - if (!StringUtils.isEmpty(clientId)) { - props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - } - this.settings = settings; - this.producer = new KafkaProducer<>(props); - this.defaultTopic = defaultTopic; - this.admin = admin; - topics = ConcurrentHashMap.newKeySet(); - } - - @Override - public void init() { - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); - byte[] data = msg.getData(); - ProducerRecord record; - Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); - producer.send(record, (metadata, exception) -> { - if (exception == null) { - if (callback != null) { - callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); - } - } else { - if (callback != null) { - callback.onFailure(exception); - } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); - } - } - }); - } - - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { - return; - } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); - } - - @Override - public void stop() { - if (producer != null) { - producer.close(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java deleted file mode 100644 index 4b68abf929..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") -public class TbKafkaTopicConfigs { - @Value("${queue.kafka.topic-properties.core}") - private String coreProperties; - @Value("${queue.kafka.topic-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.kafka.topic-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.kafka.topic-properties.notifications}") - private String notificationsProperties; - @Value("${queue.kafka.topic-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreConfigs; - @Getter - private Map ruleEngineConfigs; - @Getter - private Map transportApiConfigs; - @Getter - private Map notificationsConfigs; - @Getter - private Map jsExecutorConfigs; - - @PostConstruct - private void init() { - coreConfigs = getConfigs(coreProperties); - ruleEngineConfigs = getConfigs(ruleEngineProperties); - transportApiConfigs = getConfigs(transportApiProperties); - notificationsConfigs = getConfigs(notificationsProperties); - jsExecutorConfigs = getConfigs(jsExecutorProperties); - } - - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - return configs; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java deleted file mode 100644 index f51f5b8ae0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -@Slf4j -public final class InMemoryStorage { - private static InMemoryStorage instance; - private final ConcurrentHashMap> storage; - - private InMemoryStorage() { - storage = new ConcurrentHashMap<>(); - } - - public static InMemoryStorage getInstance() { - if (instance == null) { - synchronized (InMemoryStorage.class) { - if (instance == null) { - instance = new InMemoryStorage(); - } - } - } - return instance; - } - - public boolean put(String topic, TbQueueMsg msg) { - return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); - } - - public List get(String topic) throws InterruptedException { - if (storage.containsKey(topic)) { - List entities; - T first = (T) storage.get(topic).poll(); - if (first != null) { - entities = new ArrayList<>(); - entities.add(first); - List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 999); - for (TbQueueMsg other : otherList) { - entities.add((T) other); - } - } else { - entities = Collections.emptyList(); - } - return entities; - } - return Collections.emptyList(); - } - - /** - * Used primarily for testing. - */ - public void cleanup() { - storage.clear(); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java deleted file mode 100644 index a619eda132..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class InMemoryTbQueueConsumer implements TbQueueConsumer { - private final InMemoryStorage storage = InMemoryStorage.getInstance(); - private volatile Set partitions; - private volatile boolean stopped; - private volatile boolean subscribed; - - public InMemoryTbQueueConsumer(String topic) { - this.topic = topic; - stopped = false; - } - - private final String topic; - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = true; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = true; - } - - @Override - public void unsubscribe() { - stopped = true; - } - - @Override - public List poll(long durationInMillis) { - if (subscribed) { - List messages = partitions - .stream() - .map(tpi -> { - try { - return storage.get(tpi.getFullTopicName()); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Queue was interrupted.", e); - } - return Collections.emptyList(); - } - }) - .flatMap(List::stream) - .map(msg -> (T) msg).collect(Collectors.toList()); - if (messages.size() > 0) { - return messages; - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to sleep.", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java deleted file mode 100644 index cfcd788a16..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -@Data -public class InMemoryTbQueueProducer implements TbQueueProducer { - - private final InMemoryStorage storage = InMemoryStorage.getInstance(); - - private final String defaultTopic; - - public InMemoryTbQueueProducer(String defaultTopic) { - this.defaultTopic = defaultTopic; - } - - @Override - public void init() { - - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - boolean result = storage.put(tpi.getFullTopicName(), msg); - if (result) { - if (callback != null) { - callback.onSuccess(null); - } - } else { - if (callback != null) { - callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); - } - } - } - - @Override - public void stop() { - - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java deleted file mode 100644 index 76ff04c238..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java deleted file mode 100644 index a7349091bd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") -public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbAwsSqsSettings sqsSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.sqsSettings = sqsSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java deleted file mode 100644 index 3056eced2c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") -public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbAwsSqsSettings sqsSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.sqsSettings = sqsSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java deleted file mode 100644 index 351cd4058c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbAwsSqsProducerTemplate> producerTemplate = - new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); - - TbAwsSqsConsumerTemplate> consumerTemplate = - new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, - transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java deleted file mode 100644 index fbdbeaab0c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java deleted file mode 100644 index 2855e577ee..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - InMemoryTbQueueProducer> producerTemplate = - new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - - InMemoryTbQueueConsumer> consumerTemplate = - new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - - templateBuilder.queueAdmin(new TbQueueAdmin() { - @Override - public void createTopicIfNotExists(String topic) {} - - @Override - public void destroy() {} - }); - - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java deleted file mode 100644 index 72c6f7b8b9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); - requestBuilder.admin(notificationAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - String queueName = configuration.getName(); - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(ruleEngineAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(coreAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(transportApiAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); - requestBuilder.admin(transportApiAdmin); - return requestBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java deleted file mode 100644 index ca10830981..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") -public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(coreAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(transportApiAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java deleted file mode 100644 index bcac399a64..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") -public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - String queueName = configuration.getName(); - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(ruleEngineAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java deleted file mode 100644 index 2f7aab4779..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { - - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); - requestBuilder.admin(transportApiAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - responseBuilder.admin(transportApiAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(requestBuilder.build()); - templateBuilder.responseTemplate(responseBuilder.build()); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - responseBuilder.admin(notificationAdmin); - return responseBuilder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java deleted file mode 100644 index af0b276c1a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java deleted file mode 100644 index 0a23d58e46..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") -public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java deleted file mode 100644 index 79501130ed..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") -public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java deleted file mode 100644 index 7e859e02ab..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class PubSubTransportQueueFactory implements TbTransportQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producer); - templateBuilder.responseTemplate(consumer); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java deleted file mode 100644 index 72f45e5276..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") -public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java deleted file mode 100644 index d84bbedbd2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") -public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbRabbitMqQueueArguments queueArguments) { - this.rabbitMqSettings = rabbitMqSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java deleted file mode 100644 index a18f427fab..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") -public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbRabbitMqSettings rabbitMqSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.rabbitMqSettings = rabbitMqSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java deleted file mode 100644 index 841e004b3f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbRabbitMqQueueArguments queueArguments) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbRabbitMqProducerTemplate> producerTemplate = - new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); - - TbRabbitMqConsumerTemplate> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java deleted file mode 100644 index 8b01c38607..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") -public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - - public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java deleted file mode 100644 index 523ac88bd1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") -public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; - - public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { - this.serviceBusSettings = serviceBusSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java deleted file mode 100644 index 8a2f9b396b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") -public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - - public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java deleted file mode 100644 index 33c5277ca5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueAdmin admin) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = - new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - - TbQueueConsumer> consumerTemplate = - new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java deleted file mode 100644 index 59a8f449b0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -/** - * Responsible for initialization of various Producers and Consumers used by TB Core Node. - * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable - */ -public interface TbCoreQueueFactory { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> createTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreMsgProducer(); - - /** - * Used to push notifications to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreNotificationsMsgProducer(); - - /** - * Used to consume messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToCoreMsgConsumer(); - - /** - * Used to consume high priority messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToCoreNotificationsMsgConsumer(); - - /** - * Used to consume Transport API Calls - * - * @return - */ - TbQueueConsumer> createTransportApiRequestConsumer(); - - /** - * Used to push replies to Transport API Calls - * - * @return - */ - TbQueueProducer> createTransportApiResponseProducer(); - - TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java deleted file mode 100644 index 9fe5de0bcc..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { - - private final TbCoreQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - - public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - return toTransport; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - return toRuleEngineNotifications; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - return toTbCoreNotifications; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java deleted file mode 100644 index 34c6ee577f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -/** - * Responsible for providing various Producers to other services. - */ -public interface TbQueueProducerProvider { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> getTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> getRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> getTbCoreMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java deleted file mode 100644 index 69de7ca3e2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") -public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { - - private final TbRuleEngineQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - - - public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - return toTransport; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - return toRuleEngineNotifications; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - return toTbCoreNotifications; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java deleted file mode 100644 index 561b3e84ea..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -/** - * Responsible for initialization of various Producers and Consumers used by TB Core Node. - * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable - */ -public interface TbRuleEngineQueueFactory { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> createTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreMsgProducer(); - - /** - * Used to push notifications to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreNotificationsMsgProducer(); - - /** - * Used to consume messages by TB Core Service - * - * @return - * @param configuration - */ - //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary - TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); - - /** - * Used to consume high priority messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); - - TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java deleted file mode 100644 index dc1d2c449c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -public interface TbTransportQueueFactory { - - TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); - - TbQueueProducer> createRuleEngineMsgProducer(); - - TbQueueProducer> createTbCoreMsgProducer(); - - TbQueueConsumer> createTransportNotificationsConsumer(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java deleted file mode 100644 index bced3f83ac..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") -public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { - - private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - - public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java deleted file mode 100644 index 1241370d05..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.cloud.pubsub.v1.SubscriptionAdminClient; -import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; -import com.google.cloud.pubsub.v1.TopicAdminClient; -import com.google.cloud.pubsub.v1.TopicAdminSettings; -import com.google.protobuf.Duration; -import com.google.pubsub.v1.ListSubscriptionsRequest; -import com.google.pubsub.v1.ListTopicsRequest; -import com.google.pubsub.v1.ProjectName; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.Subscription; -import com.google.pubsub.v1.Topic; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class TbPubSubAdmin implements TbQueueAdmin { - private static final String ACK_DEADLINE = "ackDeadlineInSec"; - private static final String MESSAGE_RETENTION = "messageRetentionInSec"; - - private final TopicAdminClient topicAdminClient; - private final SubscriptionAdminClient subscriptionAdminClient; - - private final TbPubSubSettings pubSubSettings; - private final Set topicSet = ConcurrentHashMap.newKeySet(); - private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); - private final Map subscriptionProperties; - - public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.subscriptionProperties = subscriptionSettings; - - TopicAdminSettings topicAdminSettings; - try { - topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create TopicAdminSettings"); - throw new RuntimeException("Failed to create TopicAdminSettings."); - } - - SubscriptionAdminSettings subscriptionAdminSettings; - try { - subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create SubscriptionAdminSettings"); - throw new RuntimeException("Failed to create SubscriptionAdminSettings."); - } - - try { - topicAdminClient = TopicAdminClient.create(topicAdminSettings); - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - topicSet.add(topic.getName()); - } - } catch (IOException e) { - log.error("Failed to get topics.", e); - throw new RuntimeException("Failed to get topics.", e); - } - - try { - subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - subscriptionSet.add(subscription.getName()); - } - } catch (IOException e) { - log.error("Failed to get subscriptions.", e); - throw new RuntimeException("Failed to get subscriptions.", e); - } - } - - @Override - public void createTopicIfNotExists(String partition) { - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); - - if (topicSet.contains(topicName.toString())) { - createSubscriptionIfNotExists(partition, topicName); - return; - } - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } - } - - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); - createSubscriptionIfNotExists(partition, topicName); - } - - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { - ProjectSubscriptionName subscriptionName = - ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); - - if (subscriptionSet.contains(subscriptionName.toString())) { - return; - } - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } - } - - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); - - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); - - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } - - private void setAckDeadline(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(ACK_DEADLINE)) { - builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); - } - } - - private void setMessageRetention(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { - Duration duration = Duration - .newBuilder() - .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) - .build(); - builder.setMessageRetentionDuration(duration); - } - } - - @Override - public void destroy() { - if (topicAdminClient != null) { - topicAdminClient.close(); - } - if (subscriptionAdminClient != null) { - subscriptionAdminClient.close(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java deleted file mode 100644 index 5dc795739a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -@Slf4j -public class TbPubSubConsumerTemplate implements TbQueueConsumer { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbPubSubSettings pubSubSettings; - - private volatile boolean subscribed; - private volatile Set partitions; - private volatile Set subscriptionNames; - private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - - private ExecutorService consumerExecutor; - private final SubscriberStub subscriber; - private volatile boolean stopped; - - private volatile int messagesPerTopic; - - public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.pubSubSettings = pubSubSettings; - this.topic = topic; - this.decoder = decoder; - - try { - SubscriberStubSettings subscriberStubSettings = - SubscriberStubSettings.newBuilder() - .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setTransportChannelProvider( - SubscriberStubSettings.defaultGrpcTransportProviderBuilder() - .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) - .build()) - .build(); - - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); - } catch (IOException e) { - log.error("Failed to create subscriber.", e); - throw new RuntimeException("Failed to create subscriber.", e); - } - stopped = false; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - - if (subscriber != null) { - subscriber.close(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); - subscriptionNames.forEach(admin::createTopicIfNotExists); - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); - subscribed = true; - } - List messages; - try { - messages = receiveMessages(); - if (!messages.isEmpty()) { - List result = new ArrayList<>(); - messages.forEach(msg -> { - try { - result.add(decode(msg.getMessage())); - } catch (InvalidProtocolBufferException e) { - log.error("Failed decode record: [{}]", msg); - } - }); - return result; - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); - acknowledgeRequests.clear(); - } - - private List receiveMessages() throws ExecutionException, InterruptedException { - List>> result = subscriptionNames.stream().map(subscriptionId -> { - String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); - PullRequest pullRequest = - PullRequest.newBuilder() - .setMaxMessages(messagesPerTopic) - .setReturnImmediately(false) // return immediately if messages are not available - .setSubscription(subscriptionName) - .build(); - - ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); - - return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { - if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { - List ackIds = new ArrayList<>(); - for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { - ackIds.add(message.getAckId()); - } - AcknowledgeRequest acknowledgeRequest = - AcknowledgeRequest.newBuilder() - .setSubscription(subscriptionName) - .addAllAckIds(ackIds) - .build(); - - acknowledgeRequests.add(acknowledgeRequest); - return pullResponse.getReceivedMessagesList(); - } - return null; - }, consumerExecutor); - - }).collect(Collectors.toList()); - - ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { - if (!CollectionUtils.isEmpty(listMessages)) { - return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - - return transform.get(); - } - - public T decode(PubsubMessage message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java deleted file mode 100644 index 2cd2e1054e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutureCallback; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.Publisher; -import com.google.gson.Gson; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PubsubMessage; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class TbPubSubProducerTemplate implements TbQueueProducer { - - private final Gson gson = new Gson(); - - private final String defaultTopic; - private final TbQueueAdmin admin; - private final TbPubSubSettings pubSubSettings; - - private final Map publisherMap = new ConcurrentHashMap<>(); - - private ExecutorService pubExecutor = Executors.newCachedThreadPool(); - - public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { - this.defaultTopic = defaultTopic; - this.admin = admin; - this.pubSubSettings = pubSubSettings; - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); - pubsubMessageBuilder.setData(getMsg(msg)); - - Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); - ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); - - ApiFutures.addCallback(future, new ApiFutureCallback() { - public void onSuccess(String messageId) { - if (callback != null) { - callback.onSuccess(null); - } - } - - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }, pubExecutor); - } - - @Override - public void stop() { - publisherMap.forEach((k, v) -> { - if (v != null) { - try { - v.shutdown(); - v.awaitTermination(1, TimeUnit.SECONDS); - } catch (Exception e) { - log.error("Failed to shutdown PubSub client during destroy()", e); - } - } - }); - - if (pubExecutor != null) { - pubExecutor.shutdownNow(); - } - } - - private ByteString getMsg(T msg) { - String json = gson.toJson(new DefaultTbQueueMsg(msg)); - return ByteString.copyFrom(json.getBytes()); - } - - private Publisher getOrCreatePublisher(String topic) { - if (publisherMap.containsKey(topic)) { - return publisherMap.get(topic); - } else { - try { - admin.createTopicIfNotExists(topic); - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); - Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - publisherMap.put(topic, publisher); - return publisher; - } catch (IOException e) { - log.error("Failed to create topic [{}].", topic, e); - throw new RuntimeException("Failed to create topic.", e); - } - } - - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java deleted file mode 100644 index d6c6d0e271..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; -import com.google.auth.oauth2.ServiceAccountCredentials; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.io.ByteArrayInputStream; -import java.io.IOException; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -@Component -@Data -public class TbPubSubSettings { - - @Value("${queue.pubsub.project_id}") - private String projectId; - - @Value("${queue.pubsub.service_account}") - private String serviceAccount; - - @Value("${queue.pubsub.max_msg_size}") - private int maxMsgSize; - - @Value("${queue.pubsub.max_messages}") - private int maxMessages; - - private CredentialsProvider credentialsProvider; - - @PostConstruct - private void init() throws IOException { - ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( - new ByteArrayInputStream(serviceAccount.getBytes())); - credentialsProvider = FixedCredentialsProvider.create(credentials); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java deleted file mode 100644 index e98f51ec7c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -public class TbPubSubSubscriptionSettings { - @Value("${queue.pubsub.queue-properties.core}") - private String coreProperties; - @Value("${queue.pubsub.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.pubsub.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.pubsub.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.pubsub.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreSettings; - @Getter - private Map ruleEngineSettings; - @Getter - private Map transportApiSettings; - @Getter - private Map notificationsSettings; - @Getter - private Map jsExecutorSettings; - - @PostConstruct - private void init() { - coreSettings = getSettings(coreProperties); - ruleEngineSettings = getSettings(ruleEngineProperties); - transportApiSettings = getSettings(transportApiProperties); - notificationsSettings = getSettings(notificationsProperties); - jsExecutorSettings = getSettings(jsExecutorProperties); - } - - private Map getSettings(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - return configs; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java deleted file mode 100644 index 3bef6deb84..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeoutException; - -@Slf4j -public class TbRabbitMqAdmin implements TbQueueAdmin { - - private final TbRabbitMqSettings rabbitMqSettings; - private final Channel channel; - private final Connection connection; - private final Map arguments; - - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { - this.rabbitMqSettings = rabbitMqSettings; - this.arguments = arguments; - - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic) { - try { - channel.queueDeclare(topic, false, false, false, arguments); - } catch (IOException e) { - log.error("Failed to bind queue: [{}]", topic, e); - } - } - - @Override - public void destroy() { - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close Chanel.", e); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close Connection.", e); - } - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java deleted file mode 100644 index 25d7719163..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.GetResponse; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -@Slf4j -public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbRabbitMqSettings rabbitMqSettings; - private final Channel channel; - private final Connection connection; - - private volatile Set partitions; - private volatile boolean subscribed; - private volatile Set queues; - private volatile boolean stopped; - - public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.rabbitMqSettings = rabbitMqSettings; - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - stopped = false; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - - queues.forEach(admin::createTopicIfNotExists); - subscribed = true; - } - - List result = queues.stream() - .map(queue -> { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - }).filter(Objects::nonNull).map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to decode message: [{}].", message); - throw new RuntimeException("Failed to decode message.", e); - } - }).collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to wait.", e); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - - public T decode(GetResponse message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java deleted file mode 100644 index 91b46213a5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.rabbitmq.client.AMQP; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; - -@Slf4j -public class TbRabbitMqProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbRabbitMqSettings rabbitMqSettings; - private ListeningExecutorService producerExecutor; - private final Channel channel; - private final Connection connection; - - public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.rabbitMqSettings = rabbitMqSettings; - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - AMQP.BasicProperties properties = new AMQP.BasicProperties(); - try { - channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); - if (callback != null) { - callback.onSuccess(null); - } - } catch (IOException e) { - log.error("Failed publish message: [{}].", msg, e); - if (callback != null) { - callback.onFailure(e); - } - } - } - - @Override - public void stop() { - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java deleted file mode 100644 index eb73e7dce6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -public class TbRabbitMqQueueArguments { - @Value("${queue.rabbitmq.queue-properties.core}") - private String coreProperties; - @Value("${queue.rabbitmq.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.rabbitmq.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.rabbitmq.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.rabbitmq.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreArgs; - @Getter - private Map ruleEngineArgs; - @Getter - private Map transportApiArgs; - @Getter - private Map notificationsArgs; - @Getter - private Map jsExecutorArgs; - - @PostConstruct - private void init() { - coreArgs = getArgs(coreProperties); - ruleEngineArgs = getArgs(ruleEngineProperties); - transportApiArgs = getArgs(transportApiProperties); - notificationsArgs = getArgs(notificationsProperties); - jsExecutorArgs = getArgs(jsExecutorProperties); - } - - private Map getArgs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String strValue = property.substring(delimiterPosition + 1); - configs.put(key, getObjectValue(strValue)); - } - return configs; - } - - private Object getObjectValue(String str) { - if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { - return Boolean.valueOf(str); - } else if (isNumeric(str)) { - return getNumericValue(str); - } - return str; - } - - private Object getNumericValue(String str) { - if (str.contains(".")) { - return Double.valueOf(str); - } else { - return Long.valueOf(str); - } - } - - private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - - public boolean isNumeric(String strNum) { - if (strNum == null) { - return false; - } - return PATTERN.matcher(strNum).matches(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java deleted file mode 100644 index e0156e6dc8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.ConnectionFactory; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -@Component -@Data -public class TbRabbitMqSettings { - @Value("${queue.rabbitmq.exchange_name:}") - private String exchangeName; - @Value("${queue.rabbitmq.host:}") - private String host; - @Value("${queue.rabbitmq.port:}") - private int port; - @Value("${queue.rabbitmq.virtual_host:}") - private String virtualHost; - @Value("${queue.rabbitmq.username:}") - private String username; - @Value("${queue.rabbitmq.password:}") - private String password; - @Value("${queue.rabbitmq.automatic_recovery_enabled:}") - private boolean automaticRecoveryEnabled; - @Value("${queue.rabbitmq.connection_timeout:}") - private int connectionTimeout; - @Value("${queue.rabbitmq.handshake_timeout:}") - private int handshakeTimeout; - - private ConnectionFactory connectionFactory; - - @PostConstruct - private void init() { - connectionFactory = new ConnectionFactory(); - connectionFactory.setHost(host); - connectionFactory.setPort(port); - connectionFactory.setVirtualHost(virtualHost); - connectionFactory.setUsername(username); - connectionFactory.setPassword(password); - connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); - connectionFactory.setConnectionTimeout(connectionTimeout); - connectionFactory.setHandshakeTimeout(handshakeTimeout); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java deleted file mode 100644 index 1c260c754d..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueCoreSettings { - - @Value("${queue.core.topic}") - private String topic; - - @Value("${queue.core.partitions}") - private int partitions; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java deleted file mode 100644 index cd492e87f1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueRemoteJsInvokeSettings { - @Value("${queue.js.request_topic}") - private String requestTopic; - - @Value("${queue.js.response_topic_prefix}") - private String responseTopic; - - @Value("${queue.js.max_pending_requests}") - private long maxPendingRequests; - - @Value("${queue.js.response_poll_interval}") - private int responsePollInterval; - - @Value("${queue.js.response_auto_commit_interval}") - private int autoCommitInterval; - - @Value("${queue.js.max_requests_timeout}") - private long maxRequestsTimeout; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java deleted file mode 100644 index 9dfe715ece..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.PostConstruct; -import java.util.List; - -@Slf4j -@Data -@EnableAutoConfiguration -@Configuration -@ConfigurationProperties(prefix = "queue.rule-engine") -public class TbQueueRuleEngineSettings { - - private String topic; - private List queues; - - //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. - // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers - @PostConstruct - public void validate() { - queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { - log.error("Main queue is not configured in thingsboard.yml"); - return new RuntimeException("No \"Main\" queue configured!"); - }); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java deleted file mode 100644 index 5ee58ab17e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueTransportApiSettings { - - @Value("${queue.transport_api.requests_topic}") - private String requestsTopic; - - @Value("${queue.transport_api.responses_topic}") - private String responsesTopic; - - @Value("${queue.transport_api.max_pending_requests}") - private int maxPendingRequests; - - @Value("${queue.transport_api.max_requests_timeout}") - private int maxRequestsTimeout; - - @Value("${queue.transport_api.max_callback_threads}") - private int maxCallbackThreads; - - @Value("${queue.transport_api.request_poll_interval}") - private long requestPollInterval; - - @Value("${queue.transport_api.response_poll_interval}") - private long responsePollInterval; - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java deleted file mode 100644 index 019a76953b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; - -@Data -public class TbRuleEngineQueueConfiguration { - - private String name; - private String topic; - private int pollInterval; - private int partitions; - private long packProcessingTimeout; - private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; - private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java deleted file mode 100644 index eb4eaed7ed..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.http.SdkHttpMetadata; -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -@Data -@AllArgsConstructor -public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { - - private final SdkHttpMetadata metadata; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java deleted file mode 100644 index 8e293e6dff..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.CreateQueueRequest; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -public class TbAwsSqsAdmin implements TbQueueAdmin { - - private final Map attributes; - private final AmazonSQS sqsClient; - private final Set queues; - - public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { - this.attributes = attributes; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) - .withRegion(sqsSettings.getRegion()) - .build(); - - queues = sqsClient - .listQueues() - .getQueueUrls() - .stream() - .map(this::getQueueNameFromUrl) - .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); - } - - @Override - public void createTopicIfNotExists(String topic) { - String queueName = topic.replaceAll("\\.", "_") + ".fifo"; - if (queues.contains(queueName)) { - return; - } - final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); - String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); - queues.add(getQueueNameFromUrl(queueUrl)); - } - - private String getQueueNameFromUrl(String queueUrl) { - int delimiterIndex = queueUrl.lastIndexOf("/"); - return queueUrl.substring(delimiterIndex + 1); - } - - @Override - public void destroy() { - if (sqsClient != null) { - sqsClient.shutdown(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java deleted file mode 100644 index 3e71388844..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; -import com.amazonaws.services.sqs.model.Message; -import com.amazonaws.services.sqs.model.ReceiveMessageRequest; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { - - private static final int MAX_NUM_MSGS = 10; - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final AmazonSQS sqsClient; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbAwsSqsSettings sqsSettings; - - private final List pendingMessages = new CopyOnWriteArrayList<>(); - private volatile Set queueUrls; - private volatile Set partitions; - private ListeningExecutorService consumerExecutor; - private volatile boolean subscribed; - private volatile boolean stopped = false; - - public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.sqsSettings = sqsSettings; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); - - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - - if (sqsClient != null) { - sqsClient.shutdown(); - } - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); - subscribed = true; - } - - if (!pendingMessages.isEmpty()) { - log.warn("Present {} non committed messages.", pendingMessages.size()); - return Collections.emptyList(); - } - - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .map(msg -> { - try { - return decode(msg); - } catch (IOException e) { - log.error("Failed to decode message: [{}]", msg); - return null; - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", topic); - } else { - log.error("Failed to pool messages.", e); - } - } - } - return Collections.emptyList(); - } - - private ListenableFuture> poll(String url, int waitTimeSeconds) { - List>> result = new ArrayList<>(); - - for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { - result.add(consumerExecutor.submit(() -> { - ReceiveMessageRequest request = new ReceiveMessageRequest(); - request - .withWaitTimeSeconds(waitTimeSeconds) - .withMessageAttributeNames("headers") - .withQueueUrl(url) - .withMaxNumberOfMessages(MAX_NUM_MSGS); - return sqsClient.receiveMessage(request).getMessages(); - })); - } - return Futures.transform(Futures.allAsList(result), list -> { - if (!CollectionUtils.isEmpty(list)) { - return list.stream() - .flatMap(messageList -> { - if (!messageList.isEmpty()) { - this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); - return messageList.stream(); - } - return Stream.empty(); - }) - .collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - } - - @Override - public void commit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - - pendingMessages.clear(); - } - - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - - @Data - private static class AwsSqsMsgWrapper { - private final String url; - private final List messages; - - public AwsSqsMsgWrapper(String url, List messages) { - this.url = url; - this.messages = messages; - } - } - - private String getQueueUrl(String topic) { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java deleted file mode 100644 index 2d85539184..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.SendMessageRequest; -import com.amazonaws.services.sqs.model.SendMessageResult; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; - -@Slf4j -public class TbAwsSqsProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final AmazonSQS sqsClient; - private final Gson gson = new Gson(); - private final Map queueUrlMap = new ConcurrentHashMap<>(); - private final TbQueueAdmin admin; - private ListeningExecutorService producerExecutor; - - public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); - - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - SendMessageRequest sendMsgRequest = new SendMessageRequest(); - sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - - sendMsgRequest.withMessageGroupId(msg.getKey().toString()); - ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); - - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(SendMessageResult result) { - if (callback != null) { - callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); - } - } - - @Override - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }, producerExecutor); - } - - @Override - public void stop() { - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - if (sqsClient != null) { - sqsClient.shutdown(); - } - } - - private String getQueueUrl(String topic) { - return queueUrlMap.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java deleted file mode 100644 index 70a9587bef..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.services.sqs.model.QueueAttributeName; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") -public class TbAwsSqsQueueAttributes { - @Value("${queue.aws-sqs.queue-properties.core}") - private String coreProperties; - @Value("${queue.aws-sqs.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.aws-sqs.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.aws-sqs.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.aws-sqs.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreAttributes; - @Getter - private Map ruleEngineAttributes; - @Getter - private Map transportApiAttributes; - @Getter - private Map notificationsAttributes; - @Getter - private Map jsExecutorAttributes; - - private final Map defaultAttributes = new HashMap<>(); - - @PostConstruct - private void init() { - defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); - - coreAttributes = getConfigs(coreProperties); - ruleEngineAttributes = getConfigs(ruleEngineProperties); - transportApiAttributes = getConfigs(transportApiProperties); - notificationsAttributes = getConfigs(notificationsProperties); - jsExecutorAttributes = getConfigs(jsExecutorProperties); - } - - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - validateAttributeName(key); - configs.put(key, value); - } - configs.putAll(defaultAttributes); - return configs; - } - - private void validateAttributeName(String key) { - QueueAttributeName.fromValue(key); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java deleted file mode 100644 index ca2316e174..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public @interface TbCoreComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java deleted file mode 100644 index 3b99c94dea..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") -public @interface TbRuleEngineComponent { -} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto deleted file mode 100644 index 51bc7649be..0000000000 --- a/common/queue/src/main/proto/queue.proto +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package transport; - -option java_package = "org.thingsboard.server.gen.transport"; -option java_outer_classname = "TransportProtos"; - -message QueueInfo { - string name = 1; - string topic = 2; - int32 partitions = 3; -} - -/** - * Service Discovery Data Structures; - */ -message ServiceInfo { - string serviceId = 1; - repeated string serviceTypes = 2; - int64 tenantIdMSB = 3; - int64 tenantIdLSB = 4; - repeated QueueInfo ruleEngineQueues = 5; -} - -/** - * Transport Service Data Structures; - */ -message SessionInfoProto { - string nodeId = 1; - int64 sessionIdMSB = 2; - int64 sessionIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - int64 deviceIdMSB = 6; - int64 deviceIdLSB = 7; - string deviceName = 8; - string deviceType = 9; -} - -enum SessionEvent { - OPEN = 0; - CLOSED = 1; -} - -enum SessionType { - SYNC = 0; - ASYNC = 1; -} - -enum KeyValueType { - BOOLEAN_V = 0; - LONG_V = 1; - DOUBLE_V = 2; - STRING_V = 3; - JSON_V = 4; -} - -message KeyValueProto { - string key = 1; - KeyValueType type = 2; - bool bool_v = 3; - int64 long_v = 4; - double double_v = 5; - string string_v = 6; - string json_v = 7; -} - -message TsKvProto { - int64 ts = 1; - KeyValueProto kv = 2; -} - -message TsKvListProto { - int64 ts = 1; - repeated KeyValueProto kv = 2; -} - -message DeviceInfoProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - string deviceName = 5; - string deviceType = 6; - string additionalInfo = 7; -} - -/** - * Transport Service Messages; - */ -message SessionEventMsg { - SessionType sessionType = 1; - SessionEvent event = 2; -} - -message PostTelemetryMsg { - repeated TsKvListProto tsKvList = 1; -} - -message PostAttributeMsg { - repeated KeyValueProto kv = 1; -} - -message GetAttributeRequestMsg { - int32 requestId = 1; - repeated string clientAttributeNames = 2; - repeated string sharedAttributeNames = 3; -} - -message GetAttributeResponseMsg { - int32 requestId = 1; - repeated TsKvProto clientAttributeList = 2; - repeated TsKvProto sharedAttributeList = 3; - repeated string deletedAttributeKeys = 4; - string error = 5; -} - -message AttributeUpdateNotificationMsg { - repeated TsKvProto sharedUpdated = 1; - repeated string sharedDeleted = 2; -} - -message ValidateDeviceTokenRequestMsg { - string token = 1; -} - -message ValidateDeviceX509CertRequestMsg { - string hash = 1; -} - -message ValidateDeviceCredentialsResponseMsg { - DeviceInfoProto deviceInfo = 1; - string credentialsBody = 2; -} - -message GetOrCreateDeviceFromGatewayRequestMsg { - int64 gatewayIdMSB = 1; - int64 gatewayIdLSB = 2; - string deviceName = 3; - string deviceType = 4; -} - -message GetOrCreateDeviceFromGatewayResponseMsg { - DeviceInfoProto deviceInfo = 1; -} - -message GetTenantRoutingInfoRequestMsg { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; -} - -message GetTenantRoutingInfoResponseMsg { - bool isolatedTbCore = 1; - bool isolatedTbRuleEngine = 2; -} - -message SessionCloseNotificationProto { - string message = 1; -} - -message SubscribeToAttributeUpdatesMsg { - bool unsubscribe = 1; -} - -message SubscribeToRPCMsg { - bool unsubscribe = 1; -} - -message ToDeviceRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToDeviceRpcResponseMsg { - int32 requestId = 1; - string payload = 2; -} - -message ToServerRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToServerRpcResponseMsg { - int32 requestId = 1; - string payload = 2; - string error = 3; -} - -message ClaimDeviceMsg { - int64 deviceIdMSB = 1; - int64 deviceIdLSB = 2; - string secretKey = 3; - int64 durationMs = 4; -} - -//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. -message SubscriptionInfoProto { - int64 lastActivityTime = 1; - bool attributeSubscription = 2; - bool rpcSubscription = 3; -} - -message SessionSubscriptionInfoProto { - SessionInfoProto sessionInfo = 1; - SubscriptionInfoProto subscriptionInfo = 2; -} - -message DeviceSessionsCacheEntry { - repeated SessionSubscriptionInfoProto sessions = 1; -} - -message TransportToDeviceActorMsg { - SessionInfoProto sessionInfo = 1; - SessionEventMsg sessionEvent = 2; - GetAttributeRequestMsg getAttributes = 3; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; - SubscribeToRPCMsg subscribeToRPC = 5; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; - SubscriptionInfoProto subscriptionInfo = 7; - ClaimDeviceMsg claimDevice = 8; -} - -message TransportToRuleEngineMsg { - SessionInfoProto sessionInfo = 1; - PostTelemetryMsg postTelemetry = 2; - PostAttributeMsg postAttributes = 3; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; - ToServerRpcRequestMsg toServerRPCCallRequest = 5; -} - -/** - * TB Core Data Structures - */ - -message TbSubscriptionProto { - string serviceId = 1; - string sessionId = 2; - int32 subscriptionId = 3; - string entityType = 4; - int64 tenantIdMSB = 5; - int64 tenantIdLSB = 6; - int64 entityIdMSB = 7; - int64 entityIdLSB = 8; -} - -message TbTimeSeriesSubscriptionProto { - TbSubscriptionProto sub = 1; - bool allKeys = 2; - repeated TbSubscriptionKetStateProto keyStates = 3; - int64 startTime = 4; - int64 endTime = 5; -} - -message TbAttributeSubscriptionProto { - TbSubscriptionProto sub = 1; - bool allKeys = 2; - repeated TbSubscriptionKetStateProto keyStates = 3; - string scope = 4; -} - -message TbSubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated TbSubscriptionUpdateValueListProto data = 5; -} - -message TbAttributeUpdateProto { - string entityType = 1; - int64 entityIdMSB = 2; - int64 entityIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - string scope = 6; - repeated TsKvProto data = 7; -} - -message TbTimeSeriesUpdateProto { - string entityType = 1; - int64 entityIdMSB = 2; - int64 entityIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - repeated TsKvProto data = 6; -} - -message TbSubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message TbSubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message TbSubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} - -/** - * TB Core to TB Core messages - */ - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - -message SubscriptionMgrMsgProto { - TbTimeSeriesSubscriptionProto telemetrySub = 1; - TbAttributeSubscriptionProto attributeSub = 2; - TbSubscriptionCloseProto subClose = 3; - TbTimeSeriesUpdateProto tsUpdate = 4; - TbAttributeUpdateProto attrUpdate = 5; -} - -message LocalSubscriptionServiceMsgProto { - TbSubscriptionUpdateProto subUpdate = 1; -} - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} -/** - * Main messages; - */ - -/* Request from Transport Service to ThingsBoard Core Service */ -message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; - GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; -} - -/* Response from ThingsBoard Core Service to Transport Service */ -message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; - GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; -} - -/* Messages that are handled by ThingsBoard Core Service */ -message ToCoreMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; - DeviceStateServiceMsgProto deviceStateServiceMsg = 2; - SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; - bytes toDeviceActorNotificationMsg = 4; -} - -/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ -message ToCoreNotificationMsg { - LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; - FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; - bytes componentLifecycleMsg = 3; -} - -/* Messages that are handled by ThingsBoard RuleEngine Service */ -message ToRuleEngineMsg { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - bytes tbMsg = 3; - repeated string relationTypes = 4; - string failureMessage = 5; -} - -message ToRuleEngineNotificationMsg { - bytes componentLifecycleMsg = 1; - FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; -} - -/* Messages that are handled by ThingsBoard Transport Service */ -message ToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 864e7b2ab9..5be2f76258 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 82678fde5d..9d2cff5c09 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,8 +303,6 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index fd4c8e2d95..d725d47a5f 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,8 +20,14 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.transport.SessionMsgProcessor; +import org.thingsboard.server.common.transport.auth.DeviceAuthService; +import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -30,7 +36,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index c2f171e39b..9df658224a 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { @@ -221,8 +221,6 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 28c38bdda6..3d344d8dc5 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index b058a1c260..107639f302 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,23 +15,33 @@ */ package org.thingsboard.server.transport.mqtt; +import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; +import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + /** * Created by ashvayka on 04.10.18. */ @Slf4j +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 7295cb786b..84c7598d9b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.DefaultTransportService; +import org.thingsboard.server.common.transport.service.AbstractTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -410,7 +410,13 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - doDisconnect(); + if (deviceSessionCtx.isConnected()) { + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + } } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -479,17 +485,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { - doDisconnect(); - } - - private void doDisconnect() { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - deviceSessionCtx.setDisconnected(); } } @@ -507,10 +505,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index fb8a0b70be..8f752b5066 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index b4b5f98238..299f5449d9 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,8 +44,6 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) - .setDeviceName(deviceInfo.getDeviceName()) - .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index de5f82f324..e0a944ed60 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.DefaultTransportService; +import org.thingsboard.server.common.transport.service.AbstractTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index b95bc4e54f..16bc802cd3 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,10 +36,6 @@ - - org.thingsboard.common - queue - org.thingsboard.common data @@ -103,6 +99,19 @@ org.apache.commons commons-lang3 + + com.google.protobuf + protobuf-java + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java similarity index 77% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java index bb541fb626..ed1bbf18b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.common.transport; -public interface TbQueueAdmin { +import org.thingsboard.server.common.data.Device; - void createTopicIfNotExists(String topic); +public interface SessionMsgProcessor { + + void onDeviceAdded(Device device); - void destroy(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index c49b9c5bd3..ab67beee0b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,9 +20,7 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -34,16 +32,15 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") -public abstract class TransportContext { +public class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; + @Autowired - private TbServiceInfoProvider serviceInfoProvider; + private TbNodeIdProvider nodeIdProvider; @Getter private ExecutorService executor; @@ -61,7 +58,7 @@ public abstract class TransportContext { } public String getNodeId() { - return serviceInfoProvider.getServiceId(); + return nodeIdProvider.getNodeId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 2af55c9206..b0772de945 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,19 +15,15 @@ */ package org.thingsboard.server.common.transport; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -39,16 +35,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { - GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); - void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -68,7 +62,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java new file mode 100644 index 0000000000..77c45e1714 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -0,0 +1,327 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 17.10.18. + */ +@Slf4j +public abstract class AbstractTransportService implements TransportService { + + @Value("${transport.rate_limits.enabled}") + private boolean rateLimitEnabled; + @Value("${transport.rate_limits.tenant}") + private String perTenantLimitsConf; + @Value("${transport.rate_limits.device}") + private String perDevicesLimitsConf; + @Value("${transport.sessions.inactivity_timeout}") + private long sessionInactivityTimeout; + @Value("${transport.sessions.report_timeout}") + private long sessionReportTimeout; + + protected ScheduledExecutorService schedulerExecutor; + protected ExecutorService transportCallbackExecutor; + + private ConcurrentMap sessions = new ConcurrentHashMap<>(); + + //TODO: Implement cleanup of this maps. + private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + + @Override + public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { + sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, + TransportServiceCallback callback) { + registerClaimingInfo(sessionInfo, msg, callback); + } + + @Override + public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + reportActivityInternal(sessionInfo); + } + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); + + protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); + + private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { + UUID sessionId = toId(sessionInfo); + SessionMetaData sessionMetaData = sessions.get(sessionId); + if (sessionMetaData != null) { + sessionMetaData.updateLastActivityTime(); + } + return sessionMetaData; + } + + private void checkInactivityAndReportActivity() { + long expTime = System.currentTimeMillis() - sessionInactivityTimeout; + sessions.forEach((uuid, sessionMD) -> { + if (sessionMD.getLastActivityTime() < expTime) { + if (log.isDebugEnabled()) { + log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + } + process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + sessions.remove(uuid); + sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + } else { + if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { + final long lastActivityTime = sessionMD.getLastActivityTime(); + process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) + .setRpcSubscription(sessionMD.isSubscribedToRPC()) + .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { + @Override + public void onSuccess(Void msg) { + sessionMD.setLastReportedActivityTime(lastActivityTime); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to report last activity time", uuid, e); + } + }); + } + } + }); + } + + @Override + public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { + SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); + sessions.putIfAbsent(toId(sessionInfo), currentSession); + + ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + deregisterSession(sessionInfo); + }, timeout, TimeUnit.MILLISECONDS); + + currentSession.setScheduledFuture(executorFuture); + } + + @Override + public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { + SessionMetaData currentSession = sessions.get(toId(sessionInfo)); + if (currentSession != null && currentSession.hasScheduledFuture()) { + log.debug("Stopping scheduler to avoid resending response if request has been ack."); + currentSession.getScheduledFuture().cancel(false); + } + sessions.remove(toId(sessionInfo)); + } + + @Override + public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + if (!rateLimitEnabled) { + return true; + } + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.TENANT)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); + } + return false; + } + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.DEVICE)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); + } + return false; + } + + return true; + } + + protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { + UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); + SessionMetaData md = sessions.get(sessionId); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + if (toSessionMsg.hasGetAttributesResponse()) { + listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); + } + if (toSessionMsg.hasAttributeUpdateNotification()) { + listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); + } + if (toSessionMsg.hasSessionCloseNotification()) { + listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); + } + if (toSessionMsg.hasToDeviceRequest()) { + listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); + } + if (toSessionMsg.hasToServerResponse()) { + listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); + } + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + //TODO: should we notify the device actor about missed session? + log.debug("[{}] Missing session.", sessionId); + } + } + + protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); + } + + protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); + } + + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + } + + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java deleted file mode 100644 index c820167de1..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ /dev/null @@ -1,600 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.util.JsonUtils; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.AsyncCallbackTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 17.10.18. - */ -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") -public class DefaultTransportService implements TransportService { - - @Value("${transport.rate_limits.enabled}") - private boolean rateLimitEnabled; - @Value("${transport.rate_limits.tenant}") - private String perTenantLimitsConf; - @Value("${transport.rate_limits.device}") - private String perDevicesLimitsConf; - @Value("${transport.sessions.inactivity_timeout}") - private long sessionInactivityTimeout; - @Value("${transport.sessions.report_timeout}") - private long sessionReportTimeout; - @Value("${transport.client_side_rpc.timeout:60000}") - private long clientSideRpcTimeout; - @Value("${queue.transport.poll_interval}") - private int notificationsPollDuration; - - private final Gson gson = new Gson(); - private final TbTransportQueueFactory queueProvider; - private final TbQueueProducerProvider producerProvider; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; - protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; - protected TbQueueConsumer> transportNotificationsConsumer; - - protected ScheduledExecutorService schedulerExecutor; - protected ExecutorService transportCallbackExecutor; - - private final ConcurrentMap sessions = new ConcurrentHashMap<>(); - private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); - //TODO: Implement cleanup of this maps. - private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); - private volatile boolean stopped = false; - - public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.queueProvider = queueProvider; - this.producerProvider = producerProvider; - this.partitionService = partitionService; - } - - @PostConstruct - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); - ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); - transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); - transportApiRequestTemplate.init(); - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> records = transportNotificationsConsumer.poll(notificationsPollDuration); - if (records.size() == 0) { - continue; - } - records.forEach(record -> { - try { - processToTransportMsg(record.getValue()); - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - transportNotificationsConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - }); - } - - @PreDestroy - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - stopped = true; - - if (transportNotificationsConsumer != null) { - transportNotificationsConsumer.unsubscribe(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (transportApiRequestTemplate != null) { - transportApiRequestTemplate.stop(); - } - } - - @Override - public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); - } - - @Override - public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { - TbProtoQueueMsg protoMsg = - new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); - try { - TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); - return response.getValue().getGetTenantRoutingInfoResponseMsg(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - - @Override - public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); - } - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build(), callback); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); - for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - metaData.putValue("ts", tsKv.getTs() + ""); - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, packCallback); - } - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build(), callback); - } - } - - private void processTimeout(String requestId) { - RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - SessionMetaData md = sessions.get(data.getSessionId()); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - TransportProtos.ToServerRpcResponseMsg responseMsg = - TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(data.getRequestId()) - .setError("timeout").build(); - listener.onToServerRpcResponse(responseMsg); - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - log.debug("[{}] Missing session.", data.getSessionId()); - } - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - UUID sessionId = toSessionId(sessionInfo); - TenantId tenantId = getTenantId(sessionInfo); - DeviceId deviceId = getDeviceId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", msg.getMethodName()); - json.add("params", JsonUtils.parse(msg.getParams())); - - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - metaData.putValue("requestId", Integer.toString(msg.getRequestId())); - metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); - metaData.putValue("sessionId", sessionId.toString()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); - - String requestId = sessionId + "-" + msg.getRequestId(); - toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); - schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build(), callback); - } - } - - @Override - public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - reportActivityInternal(sessionInfo); - } - - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toSessionId(sessionInfo); - SessionMetaData sessionMetaData = sessions.get(sessionId); - if (sessionMetaData != null) { - sessionMetaData.updateLastActivityTime(); - } - return sessionMetaData; - } - - private void checkInactivityAndReportActivity() { - long expTime = System.currentTimeMillis() - sessionInactivityTimeout; - sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getLastActivityTime() < expTime) { - if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); - } - process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); - sessions.remove(uuid); - sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - } else { - if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { - final long lastActivityTime = sessionMD.getLastActivityTime(); - process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) - .setRpcSubscription(sessionMD.isSubscribedToRPC()) - .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { - @Override - public void onSuccess(Void msg) { - sessionMD.setLastReportedActivityTime(lastActivityTime); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to report last activity time", uuid, e); - } - }); - } - } - }); - } - - @Override - public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { - SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); - - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { - listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - deregisterSession(sessionInfo); - }, timeout, TimeUnit.MILLISECONDS); - - currentSession.setScheduledFuture(executorFuture); - } - - @Override - public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); - if (currentSession != null && currentSession.hasScheduledFuture()) { - log.debug("Stopping scheduler to avoid resending response if request has been ack."); - currentSession.getScheduledFuture().cancel(false); - } - sessions.remove(toSessionId(sessionInfo)); - } - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); - } - if (!rateLimitEnabled) { - return true; - } - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.TENANT)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); - } - return false; - } - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); - } - return false; - } - - return true; - } - - protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { - UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); - SessionMetaData md = sessions.get(sessionId); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - if (toSessionMsg.hasGetAttributesResponse()) { - listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); - } - if (toSessionMsg.hasAttributeUpdateNotification()) { - listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); - } - if (toSessionMsg.hasSessionCloseNotification()) { - listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); - } - if (toSessionMsg.hasToDeviceRequest()) { - listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); - } - if (toSessionMsg.hasToServerResponse()) { - String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); - toServerRpcPendingMap.remove(requestId); - listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); - } - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - //TODO: should we notify the device actor about missed session? - log.debug("[{}] Missing session.", sessionId); - } - } - - protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - } - - protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); - } - - protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { - return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - } - - protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { - return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - } - - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); - } - - protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); - tbCoreMsgProducer.send(tpi, - new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); - } - - protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); - ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); - } - - private class TransportTbQueueCallback implements TbQueueCallback { - private final TransportServiceCallback callback; - - private TransportTbQueueCallback(TransportServiceCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); - } - - @Override - public void onFailure(Throwable t) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); - } - } - - private class MsgPackCallback implements TbQueueCallback { - private final AtomicInteger msgCount; - private final TransportServiceCallback callback; - - public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { - this.msgCount = new AtomicInteger(msgCount); - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (msgCount.decrementAndGet() <= 0) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); - } - } - - @Override - public void onFailure(Throwable t) { - callback.onError(t); - } - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java new file mode 100644 index 0000000000..17031791d5 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java @@ -0,0 +1,339 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import org.thingsboard.server.kafka.TBKafkaAdmin; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaRequestTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by ashvayka on 05.10.18. + */ +@ConditionalOnExpression("'${transport.type:null}'=='null'") +@Service +@Slf4j +public class RemoteTransportService extends AbstractTransportService { + + @Value("${kafka.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${kafka.notifications.topic}") + private String notificationsTopic; + @Value("${kafka.notifications.poll_interval}") + private int notificationsPollDuration; + @Value("${kafka.notifications.auto_commit_interval}") + private int notificationsAutoCommitInterval; + @Value("${kafka.transport_api.requests_topic}") + private String transportApiRequestsTopic; + @Value("${kafka.transport_api.responses_topic}") + private String transportApiResponsesTopic; + @Value("${kafka.transport_api.max_pending_requests}") + private long maxPendingRequests; + @Value("${kafka.transport_api.max_requests_timeout}") + private long maxRequestsTimeout; + @Value("${kafka.transport_api.response_poll_interval}") + private int responsePollDuration; + @Value("${kafka.transport_api.response_auto_commit_interval}") + private int autoCommitInterval; + + @Autowired + private TbKafkaSettings kafkaSettings; + //We use this to get the node id. We should replace this with a component that provides the node id. + @Autowired + private TbNodeIdProvider nodeIdProvider; + + private TbKafkaRequestTemplate transportApiTemplate; + private TBKafkaProducerTemplate ruleEngineProducer; + private TBKafkaConsumerTemplate mainConsumer; + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); + + private volatile boolean stopped = false; + + @PostConstruct + public void init() { + super.init(); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiRequestsTopic); + requestBuilder.encoder(new TransportApiRequestEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("transport-api-client"); + responseBuilder.autoCommit(true); + responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(new TransportApiResponseDecoder()); + + TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder + builder = TbKafkaRequestTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.maxRequestTimeout(maxRequestsTimeout); + builder.pollInterval(responsePollDuration); + transportApiTemplate = builder.build(); + transportApiTemplate.init(); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); + ruleEngineProducerBuilder.settings(kafkaSettings); + ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); + ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); + ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); + ruleEngineProducer = ruleEngineProducerBuilder.build(); + ruleEngineProducer.init(); + + String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); + + try { + TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); + CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); + result.all().get(); + } catch (Exception e) { + log.trace("Failed to create topic: {}", e.getMessage(), e); + } + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); + mainConsumerBuilder.settings(kafkaSettings); + mainConsumerBuilder.topic(notificationsTopicName); + mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); + mainConsumerBuilder.groupId("transport"); + mainConsumerBuilder.autoCommit(true); + mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); + mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); + mainConsumer = mainConsumerBuilder.build(); + mainConsumer.subscribe(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); + records.forEach(record -> { + try { + ToTransportMsg toTransportMsg = mainConsumer.decode(record); + if (toTransportMsg.hasToDeviceSessionMsg()) { + processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @PreDestroy + public void destroy() { + super.destroy(); + stopped = true; + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + + @Override + public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), + TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), + response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), + TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), + response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), + TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), + response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostTelemetry(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToServerRPCCallRequest(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { + if (callback != null) { + if (exception == null) { + this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); + } else { + this.transportCallbackExecutor.submit(() -> { + callback.onError(exception); + }); + } + } + }); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index fc1445539b..42a65440a0 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index cc154f87b2..c07e406d2e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; @@ -25,9 +24,8 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { - @Override - public ToTransportMsg decode(TbQueueMsg msg) throws IOException { - return ToTransportMsg.parseFrom(msg.getData()); + public ToTransportMsg decode(byte[] data) throws IOException { + return ToTransportMsg.parseFrom(data); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 3971fdbf83..6c4b362f03 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index 5fdc58c068..a62e696500 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; @@ -25,9 +24,8 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { - @Override - public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { - return TransportApiResponseMsg.parseFrom(msg.getData()); + public TransportApiResponseMsg decode(byte[] data) throws IOException { + return TransportApiResponseMsg.parseFrom(data); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java deleted file mode 100644 index f2534a64c9..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") -public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - - private TransportService transportService; - - @Lazy - @Autowired - public void setTransportService(TransportService transportService) { - this.transportService = transportService; - } - - @Override - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .build(); - GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); - return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index c312b3a4d9..7c590ec925 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,7 +35,6 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; - private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -43,15 +42,10 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; - this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return connected; - } - - public void setDisconnected() { - this.connected = false; + return deviceInfo != null; } } diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/transport/transport-api/src/main/proto/transport.proto new file mode 100644 index 0000000000..e8b513574a --- /dev/null +++ b/common/transport/transport-api/src/main/proto/transport.proto @@ -0,0 +1,244 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package transport; + +option java_package = "org.thingsboard.server.gen.transport"; +option java_outer_classname = "TransportProtos"; + +/** + * Data Structures; + */ +message SessionInfoProto { + string nodeId = 1; + int64 sessionIdMSB = 2; + int64 sessionIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + int64 deviceIdMSB = 6; + int64 deviceIdLSB = 7; +} + +enum SessionEvent { + OPEN = 0; + CLOSED = 1; +} + +enum SessionType { + SYNC = 0; + ASYNC = 1; +} + +enum KeyValueType { + BOOLEAN_V = 0; + LONG_V = 1; + DOUBLE_V = 2; + STRING_V = 3; + JSON_V = 4; +} + +message KeyValueProto { + string key = 1; + KeyValueType type = 2; + bool bool_v = 3; + int64 long_v = 4; + double double_v = 5; + string string_v = 6; + string json_v = 7; +} + +message TsKvProto { + int64 ts = 1; + KeyValueProto kv = 2; +} + +message TsKvListProto { + int64 ts = 1; + repeated KeyValueProto kv = 2; +} + +message DeviceInfoProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + string deviceName = 5; + string deviceType = 6; + string additionalInfo = 7; +} + +/** + * Messages that use Data Structures; + */ +message SessionEventMsg { + SessionType sessionType = 1; + SessionEvent event = 2; +} + +message PostTelemetryMsg { + repeated TsKvListProto tsKvList = 1; +} + +message PostAttributeMsg { + repeated KeyValueProto kv = 1; +} + +message GetAttributeRequestMsg { + int32 requestId = 1; + repeated string clientAttributeNames = 2; + repeated string sharedAttributeNames = 3; +} + +message GetAttributeResponseMsg { + int32 requestId = 1; + repeated TsKvProto clientAttributeList = 2; + repeated TsKvProto sharedAttributeList = 3; + repeated string deletedAttributeKeys = 4; + string error = 5; +} + +message AttributeUpdateNotificationMsg { + repeated TsKvProto sharedUpdated = 1; + repeated string sharedDeleted = 2; +} + +message ValidateDeviceTokenRequestMsg { + string token = 1; +} + +message ValidateDeviceX509CertRequestMsg { + string hash = 1; +} + +message ValidateDeviceCredentialsResponseMsg { + DeviceInfoProto deviceInfo = 1; + string credentialsBody = 2; +} + +message GetOrCreateDeviceFromGatewayRequestMsg { + int64 gatewayIdMSB = 1; + int64 gatewayIdLSB = 2; + string deviceName = 3; + string deviceType = 4; +} + +message GetOrCreateDeviceFromGatewayResponseMsg { + DeviceInfoProto deviceInfo = 1; +} + +message SessionCloseNotificationProto { + string message = 1; +} + +message SubscribeToAttributeUpdatesMsg { + bool unsubscribe = 1; +} + +message SubscribeToRPCMsg { + bool unsubscribe = 1; +} + +message ToDeviceRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToDeviceRpcResponseMsg { + int32 requestId = 1; + string payload = 2; +} + +message ToServerRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToServerRpcResponseMsg { + int32 requestId = 1; + string payload = 2; + string error = 3; +} + +message ClaimDeviceMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + string secretKey = 3; + int64 durationMs = 4; +} + +//Used to report session state to tb-node and persist this state in the cache on the tb-node level. +message SubscriptionInfoProto { + int64 lastActivityTime = 1; + bool attributeSubscription = 2; + bool rpcSubscription = 3; +} + +message SessionSubscriptionInfoProto { + SessionInfoProto sessionInfo = 1; + SubscriptionInfoProto subscriptionInfo = 2; +} + +message DeviceSessionsCacheEntry { + repeated SessionSubscriptionInfoProto sessions = 1; +} + +message TransportToDeviceActorMsg { + SessionInfoProto sessionInfo = 1; + SessionEventMsg sessionEvent = 2; + PostTelemetryMsg postTelemetry = 3; + PostAttributeMsg postAttributes = 4; + GetAttributeRequestMsg getAttributes = 5; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; + SubscribeToRPCMsg subscribeToRPC = 7; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; + ToServerRpcRequestMsg toServerRPCCallRequest = 9; + SubscriptionInfoProto subscriptionInfo = 10; + ClaimDeviceMsg claimDevice = 11; +} + +message DeviceActorToTransportMsg { + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; +} + +/** + * Main messages; + */ +message ToRuleEngineMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; +} + +message ToTransportMsg { + DeviceActorToTransportMsg toDeviceSessionMsg = 1; +} + +message TransportApiRequestMsg { + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; +} + +message TransportApiResponseMsg { + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 49926ad5c1..8148fccf79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 7080722056..5867e505b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index b5bda06a47..386220df55 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getUuid(); + UUID getId(); - void setUuid(UUID id); + void setId(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index 44fd70f977..d75262e311 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,15 +35,14 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getUuid() { + public UUID getId() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - @Override - public void setUuid(UUID id) { + public void setId(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 9f72567838..96ce14c459 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,9 +32,6 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); - // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units - public static final long EPOCH_DIFF = 122192928000000000L; - /** * Generic constants. */ @@ -115,8 +112,6 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; - public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; - public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java index 22d8b1c212..5e822a518c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -110,7 +110,7 @@ public abstract class AbstractAlarmEntity extends BaseSqlEntity public AbstractAlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setUuid(alarm.getId().getId()); + this.setId(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index fac8d68f62..4ab6144649 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -75,7 +75,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity public AbstractAssetEntity(Asset asset) { if (asset.getId() != null) { - this.setUuid(asset.getId().getId()); + this.setId(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 042fcec950..6a973335e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -68,7 +68,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti public AbstractDeviceEntity(Device device) { if (device.getId() != null) { - this.setUuid(device.getId().getId()); + this.setId(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -104,8 +104,8 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti } protected Device toDevice() { - Device device = new Device(new DeviceId(getUuid())); - device.setCreatedTime(UUIDs.unixTimestamp(getUuid())); + Device device = new Device(new DeviceId(getId())); + device.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index 3d140039ce..c5f0fd36f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -94,7 +94,7 @@ public abstract class AbstractEntityViewEntity extends Bas public AbstractEntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setUuid(entityView.getId().getId()); + this.setId(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -145,8 +145,8 @@ public abstract class AbstractEntityViewEntity extends Bas } protected EntityView toEntityView() { - EntityView entityView = new EntityView(new EntityViewId(getUuid())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getUuid())); + EntityView entityView = new EntityView(new EntityViewId(getId())); + entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); @@ -169,5 +169,4 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setAdditionalInfo(additionalInfo); return entityView; } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 1b1a27a04f..21c411d3e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -56,7 +56,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setUuid(adminSettings.getId().getId()); + this.setId(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index d05e3cdde1..fa9490747a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.alarm.Alarm; - import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Entity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index e2573ea0d1..4de738ac73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setUuid(auditLog.getId().getId()); + this.setId(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + AuditLog auditLog = new AuditLog(new AuditLogId(getId())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index d5b287fe9b..26946cf207 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,6 +34,7 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -71,7 +72,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setUuid(customer.getId().getId()); + this.setId(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(this.getUuid())); - customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Customer customer = new Customer(new CustomerId(getId())); + customer.setCreatedTime(UUIDs.unixTimestamp(getId())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 07cec4d12f..b606c35d4e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setUuid(dashboard.getId().getId()); + this.setId(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index f26cf21934..b1ecb2572b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setUuid(dashboardInfo.getId().getId()); + this.setId(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index ba9d7ce637..1d47c7d63c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setUuid(ruleChain.getUuidId()); + this.setId(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index 1a7866418a..f36edc260b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setUuid(ruleNode.getUuidId()); + this.setId(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 1da71224f0..239d3698b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,12 +72,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; - @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) - private boolean isolatedTbCore; - - @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) - private boolean isolatedTbRuleEngine; - @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -88,7 +82,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setUuid(tenant.getId().getId()); + this.setId(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -101,8 +95,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); - this.isolatedTbCore = tenant.isIsolatedTbCore(); - this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -121,8 +113,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(this.getUuid())); - tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Tenant tenant = new Tenant(new TenantId(getId())); + tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); @@ -134,8 +126,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); - tenant.setIsolatedTbCore(isolatedTbCore); - tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 6957f87961..70593ca610 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setUuid(userCredentials.getId().getId()); + this.setId(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 3af671583e..0100110daa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setUuid(user.getId().getId()); + this.setId(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(this.getUuid())); - user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + User user = new User(new UserId(getId())); + user.setCreatedTime(UUIDs.unixTimestamp(getId())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 0cb69b9d8b..8e81b30e8f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setUuid(widgetType.getId().getId()); + this.setId(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 75ea383090..7df6ea9fe7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setUuid(widgetsBundle.getId().getId()); + this.setId(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index edc9c97a9a..d3ce07f901 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getUuid() == null) { - entity.setUuid(UUIDs.timeBased()); + if (entity.getId() == null) { + entity.setId(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 076d43b0ad..2f3c72d01b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -38,7 +38,7 @@ public interface AlarmRepository extends CrudRepository { @Param("alarmType") String alarmType, Pageable pageable); - @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + + @Query("SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + "RelationEntity re " + "WHERE a.tenantId = :tenantId " + "AND a.id = re.toId AND re.toType = 'ALARM' " + @@ -51,21 +51,7 @@ public interface AlarmRepository extends CrudRepository { "AND (:idOffset IS NULL OR a.id < :idOffset) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))", - countQuery = "SELECT count(a) FROM AlarmEntity a, " + - "RelationEntity re " + - "WHERE a.tenantId = :tenantId " + - "AND a.id = re.toId AND re.toType = 'ALARM' " + - "AND re.relationTypeGroup = 'ALARM' " + - "AND re.relationType = :relationType " + - "AND re.fromId = :affectedEntityId " + - "AND re.fromType = :affectedEntityType " + - "AND (:startId IS NULL OR a.id >= :startId) " + - "AND (:endId IS NULL OR a.id <= :endId) " + - "AND (:idOffset IS NULL OR a.id < :idOffset) " + - "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") Page findAlarms(@Param("tenantId") String tenantId, @Param("affectedEntityId") String affectedEntityId, @Param("affectedEntityType") String affectedEntityType, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 56ae6a5000..002c8969da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.asset; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; @@ -42,6 +43,7 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUIDs; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/19/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index 371a627818..38abc36940 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -32,8 +32,6 @@ import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; -import javax.persistence.criteria.Predicate; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -41,7 +39,6 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.dao.DaoUtil.endTimeToId; import static org.thingsboard.server.dao.DaoUtil.startTimeToId; -import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @SqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index d8272e3a57..c634e5a064 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index c33dd4f5e1..e281bdeec4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index ab4f408da5..67f25c2325 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.component; import com.datastax.driver.core.utils.UUIDs; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -34,9 +35,11 @@ import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 0361c8d8dd..e859773bf7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.customer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; @@ -28,10 +29,12 @@ import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index e693e81ae1..7b0e6a8882 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -41,6 +41,9 @@ import java.util.UUID; @SqlDao public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao implements DashboardInfoDao { + @Autowired + private RelationDao relationDao; + @Autowired private DashboardInfoRepository dashboardInfoRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index c409f0a91f..0d8d87f74e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -17,13 +17,10 @@ package org.thingsboard.server.dao.sql.entityview; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntitySubtype; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,6 +39,7 @@ import java.util.Optional; import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Victor Basanets on 8/31/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index 8a341e0f81..59f0da89e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -69,14 +69,13 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi protected Query getQuery(EventEntity entity, String query) { return entityManager.createNativeQuery(query, EventEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) .setParameter("body", entity.getBody().toString()) .setParameter("entity_id", entity.getEntityId()) .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("ts", entity.getTs()); + .setParameter("tenant_id", entity.getTenantId()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index f5a00fe7f7..440c618667 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -40,11 +40,11 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { @Override protected EventEntity doProcessSaveOrUpdate(EventEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); + return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 528229bb33..c5a6ea2717 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -141,7 +141,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao - DaoUtil.toPageData(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable))); + DaoUtil.toPageData(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable))); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3a4cbd84e0..d2dbe725cd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.rule; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -29,9 +30,11 @@ import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; @Slf4j @Component diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 7fc55c5752..826d60b119 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -16,9 +16,11 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -28,8 +30,11 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; + /** * Created by Valerii Sosliuk on 4/30/2017. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 296730e55f..9d39a09404 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -30,6 +30,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -83,5 +84,6 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, DaoUtil.toPageable(pageLink))); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 6f7490277a..c24e3d8e27 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -29,6 +30,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; +import java.util.List; import java.util.Objects; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index fafe05957f..c4ac4e9fd8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -150,8 +150,8 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(Sort.Direction.fromString( + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( query.getOrderBy()), "ts"))); tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 0a2e210648..bf4cf5d9e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(Sort.Direction.fromString( + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index b63e4f7846..efac1cfc42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,6 +39,8 @@ import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -123,7 +125,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -137,31 +139,19 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - - @Override - protected void validateUpdate(TenantId tenantId, Tenant tenant) { - Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); - if (old == null) { - throw new DataValidationException("Can't update non existing tenant!"); - } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { - throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); - } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { - throw new DataValidationException("Can't update isolatedTbCore property!"); - } - } - }; + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 697e34930a..758aaafb10 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,7 +144,6 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -184,9 +183,7 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255), - isolated_tb_core boolean, - isolated_tb_rule_engine boolean + zip varchar(255) ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -252,4 +249,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); - diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 931856e1df..55893fc124 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,7 +144,6 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -184,9 +183,7 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255), - isolated_tb_core boolean, - isolated_tb_rule_engine boolean + zip varchar(255) ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -252,28 +249,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); - -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index bb0a964c13..32e2a78620 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index f83e80931b..3789444106 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index e0244c256e..f02ae6b577 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index 51f34a08d6..af9a4b356f 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,4 +60,3 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false -service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 781e0ee01e..c7dd8e6429 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,8 +16,6 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 -service.type=monolith - #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql @@ -35,15 +33,4 @@ service.type=monolith #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 - -queue.core.pack-processing-timeout=3000 -queue.rule-engine.pack-processing-timeout=3000 - -queue.rule-engine.queues[0].name=Main -queue.rule-engine.queues[0].topic=tb_rule_engine.main -queue.rule-engine.queues[0].poll-interval=25 -queue.rule-engine.queues[0].partitions=3 -queue.rule-engine.queues[0].pack-processing-timeout=3000 -queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES -queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f0facf8cc1..c17b1ddde4 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; -let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -44,7 +43,6 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); - headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -150,8 +148,7 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse, - headers: headers + value: rawResponse } ] } diff --git a/pom.xml b/pom.xml index a880025e3f..7cadb38031 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.2.4.RELEASE - 5.2.2.RELEASE - 5.2.2.RELEASE - 2.2.4.RELEASE - 3.1.0 + 2.1.3.RELEASE + 5.1.5.RELEASE + 5.1.4.RELEASE + 2.1.5.RELEASE + 2.9.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.10.2 - 2.10.2 - 2.10.2 + 2.9.9.3 + 2.9.9 + 2.9.9 2.2.6 - 2.13 - 2.6.3 + 2.11 + 2.4.2 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.45.Final + 4.1.37.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.64 + 1.56 2.0.1 2.5.0 2.5.3 @@ -92,15 +92,9 @@ 4.1.1 2.57 2.7.7 - 1.25 - 1.3.10 - 1.11.747 - 1.84.0 - 3.2.0 + 1.23 1.5.0 1.4.3 - 1.9.4 - 3.2.2 @@ -897,21 +891,6 @@ jts-core ${jts.version} - - com.amazonaws - aws-java-sdk-sqs - ${amazonaws.sqs.version} - - - com.google.cloud - google-cloud-pubsub - ${pubsub.client.version} - - - com.microsoft.azure - azure-servicebus - ${azure-servicebus.version} - org.passay passay @@ -922,37 +901,6 @@ uap-java ${ua-parser.version} - - commons-beanutils - commons-beanutils - ${commons-beanutils.version} - - - commons-collections - commons-collections - ${commons-collections.version} - - - org.yaml - snakeyaml - ${snakeyaml.version} - - - org.apache.struts - struts-core - ${struts.version} - - - org.apache.struts - struts-taglib - ${struts.version} - - - org.apache.struts - struts-tiles - ${struts.version} - - diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 14c3d4b996..ad58bce619 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -97,6 +97,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -1644,7 +1645,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1996,24 +1997,12 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getTimeUrlParams(TimePageLink pageLink) { - return getUrlParams(pageLink); - } - - private String getUrlParams(TimePageLink pageLink) { - return getUrlParams(pageLink, "startTime", "endTime"); - } - - private String getUrlParamsTs(TimePageLink pageLink) { - return getUrlParams(pageLink, "startTs", "endTs"); - } - - private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { - String urlParams = "limit={limit}&ascOrder={ascOrder}"; + String urlParams = this.getUrlParams(pageLink); if (pageLink.getStartTime() != null) { - urlParams += "&" + startTime + "={startTime}"; + urlParams += "&startTime={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams += "&" + endTime + "={endTime}"; + urlParams += "&endTime={endTime}"; } return urlParams; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java similarity index 54% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java rename to rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java index 4c25f27a3e..bb0210b1ad 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java @@ -13,25 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.rule.engine.api; -public interface TbCallback { +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; - TbCallback EMPTY = new TbCallback() { +import java.util.function.Consumer; - @Override - public void onSuccess() { +public interface RuleChainTransactionService { - } + void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - @Override - public void onFailure(Throwable t) { + void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - } - }; - - void onSuccess(); - - void onFailure(Throwable t); + void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 7cab689d13..34bfe4d3f4 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -29,11 +28,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { - private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originServiceId; + private final String originHost; + private final int originPort; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index baac594a17..69ce133c5d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; - -import java.util.UUID; import java.util.function.Consumer; /** @@ -25,8 +23,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); + void sendRpcReply(DeviceId deviceId, int requestId, String body); - void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index d2e19652a2..f57849fa35 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,4 +44,6 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); + void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); + } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 9b1edc5f5c..38914f4553 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -44,91 +45,27 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; -import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { - /* - * - * METHODS TO CONTROL THE MESSAGE FLOW - * - */ - - /** - * Indicates that message was successfully processed by the rule node. - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using "Success" relationType. - * - * @param msg - */ - void tellSuccess(TbMsg msg); - - /** - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using specified relationType. - * - * @param msg - * @param relationType - */ void tellNext(TbMsg msg, String relationType); - /** - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using one of specified relationTypes. - * - * @param msg - * @param relationTypes - */ + void tellNext(TbMsg msg, String relationType, Throwable th); + void tellNext(TbMsg msg, Set relationTypes); - /** - * Sends message to the current Rule Node with specified delay in milliseconds. - * Note: this message is not queued and may be lost in case of a server restart. - * - * @param msg - */ void tellSelf(TbMsg msg, long delayMs); - /** - * Notifies Rule Engine about failure to process current message. - * - * @param msg - message - * @param th - exception - */ - void tellFailure(TbMsg msg, Throwable th); - - /** - * Puts new message to queue for processing by the Root Rule Chain - * - * @param msg - message - */ - void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); - - /** - * Puts new message to custom queue for processing - * - * @param msg - message - */ - void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); - - void enqueueForTellFailure(TbMsg msg, String failureMessage); - - void enqueueForTellNext(TbMsg msg, String relationType); - - void enqueueForTellNext(TbMsg msg, Set relationTypes); - - void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - - void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + boolean isLocalEntity(EntityId entityId); - void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + void tellFailure(TbMsg msg, Throwable th); - void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + void updateSelf(RuleNode self); - void ack(TbMsg tbMsg); + void sendTbMsgToRuleEngine(TbMsg msg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -140,17 +77,8 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); - // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); - /* - * - * METHODS TO PROCESS THE MESSAGES - * - */ - - boolean isLocalEntity(EntityId entityId); - RuleNodeId getSelfId(); TenantId getTenantId(); @@ -201,7 +129,9 @@ public interface TbContext { void logJsEvalFailure(); - String getServiceId(); + String getNodeId(); + + RuleChainTransactionService getRuleChainTransactionService(); EventLoopGroup getSharedEventLoop(); @@ -209,7 +139,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); - @Deprecated RedisTemplate getRedisTemplate(); + String getServerAddress(); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 85fc75139b..26f7d14f8b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} + default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} } diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 299fdae084..1ada70091c 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,7 +35,8 @@ UTF-8 ${basedir}/../.. - 1.11.747 + 1.11.323 + 1.83.0 1.16.0 @@ -98,6 +99,7 @@ com.google.cloud google-cloud-pubsub + ${pubsub.client.version} com.google.api.grpc diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index b3344105bb..6320d5625f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import javax.script.ScriptException; + import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -61,18 +63,16 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), - throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); + ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"); + ctx.sendTbMsgToRuleEngine(ctx.alarmCreatedMsg(alarmResult.alarm, ctx.getSelfId())); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); - } else { - ctx.tellSuccess(msg); } }, - t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t) + , ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index bdf8c5fe1e..3ee2127fde 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellSuccess(msg), + m -> ctx.tellNext(msg, "Success"), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } @@ -122,9 +122,7 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), - throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.customerCreatedMsg(savedCustomer, ctx.getSelfId())); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index 3b18c540e8..5c92c7536c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -187,9 +187,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), - throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.deviceCreatedMsg(savedDevice, ctx.getSelfId())); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -204,9 +202,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), - throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.assetCreatedMsg(savedAsset, ctx.getSelfId())); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -220,9 +216,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), - throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.customerCreatedMsg(savedCustomer, ctx.getSelfId())); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index c5dc5cc5f4..f009b737e9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,14 +117,13 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (!filteredAttributes.isEmpty()) { + if (filteredAttributes != null && !filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } - ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -136,7 +135,8 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); + TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); + ctx.tellNext(updMsg, SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 2831e890d6..62410e3052 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 19d8515455..2e99e75b53 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,19 +68,18 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); - ctx.enqueueForTellNext(tbMsg, SUCCESS); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); + ctx.tellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); - ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 926c3c25f3..a583fe3c74 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -89,8 +89,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + withCallback(save(msg, ctx), aVoid -> { + ctx.tellNext(msg, SUCCESS); + }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index d43ebb26ef..59d27018d7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,8 +74,11 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, - t -> ctx.tellFailure(processException(ctx, msg, t), t)); + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + t -> { + TbMsg next = processException(ctx, msg, t); + ctx.tellFailure(next, t); + }); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 3cc6074165..37f4e4baee 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,10 +80,13 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, - t -> ctx.tellFailure(processException(ctx, msg, t), t)); + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + t -> { + TbMsg next = processException(ctx, msg, t); + ctx.tellFailure(next, t); + }); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 46ef675ce1..e16b2f2f0e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { + public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { updateGeneratorState(ctx); } @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.enqueueForTellNext(m, SUCCESS); + ctx.tellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 1807c5c4fe..0ef9c4d46a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,16 +65,15 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.enqueueForTellNext(pendingMsg, SUCCESS); + ctx.tellNext(pendingMsg, SUCCESS); } } else { - if (pendingMsgs.size() < config.getMaxPendingMsgs()) { + if(pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); - ctx.ack(msg); } else { - ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index dc0eb9f396..086e3aded0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -72,13 +72,14 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } + if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index e5e84ba0b6..59c66dfb8b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index 07a0508a87..d0fe593711 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index 5dd11c9318..486f9fbd84 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java deleted file mode 100644 index b5af3f1563..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.flow; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.common.msg.TbMsg; - -@Slf4j -@RuleNode( - type = ComponentType.ACTION, - name = "acknowledge", - configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Acknowledges the incoming message", - nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", - uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbNodeEmptyConfig" -) -public class TbAckNode implements TbNode { - - EmptyNodeConfiguration config; - - @Override - public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); - } - - @Override - public void onMsg(TbContext ctx, TbMsg msg) { - ctx.ack(msg); - ctx.tellSuccess(msg); - } - - @Override - public void destroy() { - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java deleted file mode 100644 index d5048eadbd..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.flow; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; -import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.common.msg.TbMsg; - -import static org.thingsboard.common.util.DonAsynchron.withCallback; - -@Slf4j -@RuleNode( - type = ComponentType.ACTION, - name = "checkpoint", - configClazz = TbCheckpointNodeConfiguration.class, - nodeDescription = "transfers the message to another queue", - nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", - uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeCheckPointConfig" -) -public class TbCheckpointNode implements TbNode { - - private TbCheckpointNodeConfiguration config; - - @Override - public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); - } - - @Override - public void onMsg(TbContext ctx, TbMsg msg) { - ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); - } - - @Override - public void destroy() { - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 5a8b92cf80..8e3bcdb7dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellSuccess(next); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index 832a86b5cb..fe765abdf9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,12 +47,11 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig" -) + configDirective = "tbActionNodeGpsGeofencingConfig") public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -67,7 +66,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -79,26 +78,17 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); - told = true; - } + } else if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); } } - if (!told) { - ctx.tellSuccess(msg); - } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { @@ -118,7 +108,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 4955ebc400..267e8711aa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -17,21 +17,12 @@ package org.thingsboard.rule.engine.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.*; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -39,6 +30,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Properties; +import java.util.concurrent.ExecutionException; @Slf4j @RuleNode( @@ -71,7 +63,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); @@ -81,7 +73,8 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties().forEach(properties::put); + config.getOtherProperties() + .forEach(properties::put); } addMetadataKeyValuesAsKafkaHeaders = BooleanUtils.toBooleanDefaultIfNull(config.isAddMetadataKeyValuesAsKafkaHeaders(), false); toBytesCharset = config.getKafkaHeadersCharset() != null ? Charset.forName(config.getKafkaHeadersCharset()) : StandardCharsets.UTF_8; @@ -93,7 +86,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { if (!addMetadataKeyValuesAsKafkaHeaders) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index ee46b81116..1c41c2a124 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -78,7 +79,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellSuccess(msg), + ok -> ctx.tellNext(msg, SUCCESS), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 8aa15a4f38..0bf1c23cc7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -100,7 +101,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index 43738fd706..2906d3922f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -59,7 +59,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index c7387a5ed0..bc19d6c56f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -51,7 +51,8 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback(findEntityAsync(ctx, msg.getOriginator()), + withCallback( + findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -88,7 +89,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 03f0652c3f..653b571b4b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "related device attributes", + name = "device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 12577076d2..717cb207c4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -54,10 +54,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index a3d84b25db..caef71a4db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,7 +106,8 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); + TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); + ctx.tellNext(newMsg, SUCCESS); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 7e27ec496e..6d45a3a05c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellSuccess(msg); + ctx.tellNext(msg, TbRelationTypes.SUCCESS); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 4a61cf63f3..3c3bde2915 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index ba860c103b..65eea0b4c9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -83,7 +84,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg) { + void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -94,6 +95,13 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { + if (config.isUseRedisQueueForMsgPersistence()) { + if (throwable instanceof HttpClientErrorException) { + processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); + } else { + queueProcessor.pushOnFailure(msg); + } + } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -101,9 +109,15 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { + if (config.isUseRedisQueueForMsgPersistence()) { + queueProcessor.resetCounter(); + } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellSuccess(next); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } else { + if (config.isUseRedisQueueForMsgPersistence()) { + processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); + } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -169,4 +183,11 @@ class TbHttpClient { } } + private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + if (statusCode.is4xxClientError()) { + log.warn("[{}] Client error during message delivering!", msg); + } else { + queueProcessor.pushOnFailure(msg); + } + } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java new file mode 100644 index 0000000000..cff9faaab5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.rest; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.ListOperations; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +@Slf4j +class TbRedisQueueProcessor { + + private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; + + private final TbContext ctx; + private final TbHttpClient httpClient; + private final ExecutorService executor; + private final ListOperations listOperations; + private final String redisKey; + private final boolean trimQueue; + private final int maxQueueSize; + + private AtomicInteger failuresCounter; + private Future future; + + TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { + this.ctx = ctx; + this.httpClient = httpClient; + this.executor = Executors.newSingleThreadExecutor(); + this.listOperations = ctx.getRedisTemplate().opsForList(); + this.redisKey = constructRedisKey(); + this.trimQueue = trimQueue; + this.maxQueueSize = maxQueueSize; + init(); + } + + private void init() { + failuresCounter = new AtomicInteger(0); + future = executor.submit(() -> { + while (true) { + if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { + sleep("Target HTTP server is down...", 3); + } + if (listOperations.size(redisKey) > 0) { + List list = listOperations.range(redisKey, -10, -1); + list.forEach(obj -> { + TbMsg msg = TbMsg.fromBytes((byte[]) obj); + log.debug("Trying to send the message: {}", msg); + listOperations.remove(redisKey, -1, obj); + httpClient.processMessage(ctx, msg, this); + }); + } else { + sleep("Queue is empty, waiting for tasks!", 1); + } + } + }); + } + + void destroy() { + if (future != null) { + future.cancel(true); + } + if (executor != null) { + executor.shutdownNow(); + } + } + + void push(TbMsg msg) { + listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); + if (trimQueue) { + listOperations.trim(redisKey, 0, validateMaxQueueSize()); + } + } + + void pushOnFailure(TbMsg msg) { + listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); + failuresCounter.incrementAndGet(); + } + + void resetCounter() { + failuresCounter.set(0); + } + + private String constructRedisKey() { + return ctx.getServerAddress() + ctx.getSelfId(); + } + + private int validateMaxQueueSize() { + if (maxQueueSize != 0) { + return maxQueueSize; + } + return MAX_QUEUE_SIZE; + } + + private void sleep(String logMessage, int sleepSeconds) { + try { + log.debug(logMessage); + TimeUnit.SECONDS.sleep(sleepSeconds); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted!", e); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9cb171d0dc..9b9d275977 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,6 +25,8 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.concurrent.ExecutionException; + @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -45,6 +47,7 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; + private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -52,13 +55,20 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); + if (ctx.getRedisTemplate() == null) { + throw new RuntimeException("Redis cache type must be used!"); + } + queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) { - httpClient.processMessage(ctx, msg); + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + if (useRedisQueueForMsgPersistence) { + queueProcessor.push(msg); + } else { + httpClient.processMessage(ctx, msg, null); + } } @Override @@ -66,6 +76,9 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } + if (this.queueProcessor != null) { + this.queueProcessor.destroy(); + } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 145e73d450..9444b52210 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,8 +28,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.UUID; - @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -52,22 +50,15 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); - String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); - } else if (StringUtils.isEmpty(serviceIdStr)) { - ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); - } else if (StringUtils.isEmpty(sessionIdStr)) { - ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); - ctx.tellSuccess(msg); + ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index ad1d399e76..9aa94905e6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -36,6 +38,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -83,8 +86,10 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originServiceId"); - String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; + tmp = msg.getMetaData().getValue("originHost"); + String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; + tmp = msg.getMetaData().getValue("originPort"); + int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -101,25 +106,24 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) - .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originServiceId(originServiceId) + .originHost(originHost) + .originPort(originPort) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); + TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); } }); - ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index bd3a3cdfb2..d01f28f136 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,40 +16,18 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; -import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { - public static final String SERVICE_ID = "serviceId"; - public static final String SESSION_ID = "sessionId"; - public static final String REQUEST_ID = "requestId"; - - private String serviceIdMetaDataAttribute; - private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setServiceIdMetaDataAttribute(SERVICE_ID); - configuration.setSessionIdMetaDataAttribute(SESSION_ID); - configuration.setRequestIdMetaDataAttribute(REQUEST_ID); + configuration.setRequestIdMetaDataAttribute("requestId"); return configuration; } - - public String getServiceIdMetaDataAttribute() { - return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; - } - - public String getSessionIdMetaDataAttribute() { - return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; - } - - public String getRequestIdMetaDataAttribute() { - return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; - } } - diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index a0970238d9..3ad89bc05a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,9 +61,13 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } + String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); + if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { + ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); + } } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index bc135caa5e..d735f7f93f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap.isEmpty()) { + if (tsKvMap == null) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index bd15c5bd83..d545d41d7b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,6 +22,8 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; + /** * Created by ashvayka on 02.04.18. */ @@ -32,7 +34,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 751b061666..5423236499 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,6 +25,10 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgTransactionData; + +import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -33,23 +37,37 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", + nodeDescription = "Starts synchronization of message processing based on message originator", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") -@Deprecated public class TbSynchronizationBeginNode implements TbNode { + private EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); - ctx.tellSuccess(msg); + log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); + + TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); + TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, + msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + + ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { + log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); + ctx.tellNext(startMsg, SUCCESS); + }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), + throwable -> { + log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); + ctx.tellFailure(tbMsg, throwable); + }); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index 8b83bff1fa..af3742a0fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,25 +35,30 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", + nodeDescription = "Stops synchronization of message processing based on message originator", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) -@Deprecated public class TbSynchronizationEndNode implements TbNode { + private EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); - ctx.tellSuccess(msg); + ctx.getRuleChainTransactionService().endTransaction(msg, + successMsg -> ctx.tellNext(successMsg, SUCCESS), + throwable -> ctx.tellFailure(msg, throwable)); + log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); } @Override public void destroy() { + } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index 84a6b76920..e768f699cd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellSuccess(m); + ctx.tellNext(m, SUCCESS); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 31fd207c7c..40a3e8effd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index 182a000b59..a0a1c8629f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 017bacfd72..3ff25e1e8b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 61b83d647d..67c5b1cc18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,15 +25,12 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -41,35 +38,20 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.function.Consumer; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.anyLong; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.same; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; +import static org.mockito.Mockito.*; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { @@ -84,11 +66,6 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; - @Captor - private ArgumentCaptor successCaptor; - @Captor - private ArgumentCaptor> failureCaptor; - private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -122,16 +99,15 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); + doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); - verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); - successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -165,7 +141,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -188,7 +164,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -199,8 +175,6 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); - verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); - successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -235,7 +209,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -282,7 +256,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index b0f64fb980..27feed93ba 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,6 +19,7 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -27,24 +28,17 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.same; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { @@ -64,7 +58,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -77,7 +71,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -90,7 +84,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 9286a06da6..1e752b55b8 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,14 +28,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -44,9 +40,7 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { @@ -71,7 +65,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index 70f58c5989..ace02bacd7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,13 +30,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) @@ -62,7 +62,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index a0a61ae30f..0af1cef8ae 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,19 +31,9 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -112,7 +102,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -137,7 +127,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -162,7 +152,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -176,7 +166,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); entityAttributeFetched(customerId); } @@ -187,7 +177,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -202,7 +192,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -217,7 +207,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -244,7 +234,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -256,7 +246,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(msg, SUCCESS); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -268,7 +258,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(msg, SUCCESS); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index fa845c4a18..4c1edce8f6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,7 +36,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -92,7 +91,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -120,7 +119,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -147,7 +146,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index a00e97cf1c..aa9ad8b76f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,14 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -42,10 +38,7 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) @@ -69,15 +62,15 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); - TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellSuccess(captor.capture()); + verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } @@ -91,7 +84,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); diff --git a/tools/pom.xml b/tools/pom.xml index a3018e40ce..c8058d1bd7 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.6 + 3.11.4 com.datastax.cassandra diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index ec596db10d..6dbca324e1 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..9b3572c7d1 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,143 +41,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index f44af2105e..39dc0677e8 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..4097bce831 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,143 +42,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index cd7225ad1f..ce95ee6f1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..d13a3d071e 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,20 +17,10 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Zookeeper connection parameters. Used for service discovery. -zk: - # Enable/disable zookeeper discovery service. - enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string - url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds - retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds - connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds - session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' - zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" +# Clustering properties +cluster: + # Unique id for this node (autogenerated if empty) + node_id: "${CLUSTER_NODE_ID:}" # MQTT server parameters transport: @@ -72,143 +62,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index db477c3a8c..761f04989d 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -28,7 +28,6 @@ export * from './entity-relation.service'; export * from './entity-view.service'; export * from './event.service'; export * from './http-utils'; -export * from './queue.service'; export * from './rule-chain.service'; export * from './tenant.service'; export * from './user.service'; diff --git a/ui-ngx/src/app/core/http/queue.service.ts b/ui-ngx/src/app/core/http/queue.service.ts deleted file mode 100644 index 42beadb81a..0000000000 --- a/ui-ngx/src/app/core/http/queue.service.ts +++ /dev/null @@ -1,36 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Injectable } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; -import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { Observable } from 'rxjs'; -import { ServiceType } from '@shared/models/queue.models'; - -@Injectable({ - providedIn: 'root' -}) -export class QueueService { - - constructor( - private http: HttpClient - ) { } - - public getTenantQueuesByServiceType(serviceType: ServiceType, config?: RequestConfig): Observable> { - return this.http.get>(`/api/tenant/queues?serviceType=${serviceType}`, - defaultHttpOptionsFromConfig(config)); - } -} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html index 7d0ccd5056..48c57184ab 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html @@ -56,16 +56,6 @@ -
- -
{{ 'tenant.isolated-tb-core' | translate }}
-
{{'tenant.isolated-tb-core-details' | translate}}
- - -
{{ 'tenant.isolated-tb-rule-engine' | translate }}
-
{{'tenant.isolated-tb-rule-engine-details' | translate}}
- -
diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss deleted file mode 100644 index b05c6324fa..0000000000 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host ::ng-deep { - .mat-checkbox.hinted-checkbox { - .mat-checkbox-inner-container { - margin-top: 4px; - } - } -} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts index f624a5d5cf..c8317a20c5 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts @@ -26,8 +26,7 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod @Component({ selector: 'tb-tenant', - templateUrl: './tenant.component.html', - styleUrls: ['./tenant.component.scss'] + templateUrl: './tenant.component.html' }) export class TenantComponent extends ContactBasedComponent { @@ -51,8 +50,6 @@ export class TenantComponent extends ContactBasedComponent { return this.fb.group( { title: [entity ? entity.title : '', [Validators.required]], - isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], - isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], additionalInfo: this.fb.group( { description: [entity && entity.additionalInfo ? entity.additionalInfo.description : ''] @@ -64,8 +61,6 @@ export class TenantComponent extends ContactBasedComponent { updateEntityForm(entity: Tenant) { this.entityForm.patchValue({title: entity.title}); - this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); - this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); this.entityForm.patchValue({additionalInfo: {description: entity.additionalInfo ? entity.additionalInfo.description : ''}}); } diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html deleted file mode 100644 index ccdf4a76ec..0000000000 --- a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html +++ /dev/null @@ -1,43 +0,0 @@ - - - {{ 'queue.name' | translate }} - - - - - - - - - {{ 'queue.name_required' | translate }} - - diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts deleted file mode 100644 index 4e008d5c6c..0000000000 --- a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts +++ /dev/null @@ -1,177 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; -import { Observable } from 'rxjs'; -import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; -import { TranslateService } from '@ngx-translate/core'; -import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { QueueService } from '@core/http/queue.service'; -import { ServiceType } from '@shared/models/queue.models'; - -@Component({ - selector: 'tb-queue-type-list', - templateUrl: './queue-type-list.component.html', - styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => QueueTypeListComponent), - multi: true - }] -}) -export class QueueTypeListComponent implements ControlValueAccessor, OnInit, AfterViewInit, OnDestroy { - - queueFormGroup: FormGroup; - - modelValue: string | null; - - private requiredValue: boolean; - get required(): boolean { - return this.requiredValue; - } - @Input() - set required(value: boolean) { - this.requiredValue = coerceBooleanProperty(value); - } - - @Input() - disabled: boolean; - - @Input() - queueType: ServiceType; - - @ViewChild('queueInput', {static: true}) queueInput: ElementRef; - - filteredQueues: Observable>; - - queues: Observable>; - - searchText = ''; - - private dirty = false; - - private propagateChange = (v: any) => { }; - - constructor(private store: Store, - public translate: TranslateService, - private queueService: QueueService, - private fb: FormBuilder) { - this.queueFormGroup = this.fb.group({ - queue: [null] - }); - } - - registerOnChange(fn: any): void { - this.propagateChange = fn; - } - - registerOnTouched(fn: any): void { - } - - ngOnInit() { - this.filteredQueues = this.queueFormGroup.get('queue').valueChanges - .pipe( - tap(value => { - this.updateView(value); - }), - map(value => value ? value : ''), - mergeMap(queue => this.fetchQueues(queue) ) - ); - } - - ngAfterViewInit(): void { - } - - ngOnDestroy(): void { - } - - setDisabledState(isDisabled: boolean): void { - this.disabled = isDisabled; - if (this.disabled) { - this.queueFormGroup.disable({emitEvent: false}); - } else { - this.queueFormGroup.enable({emitEvent: false}); - } - } - - writeValue(value: string | null): void { - this.searchText = ''; - this.modelValue = value; - this.queueFormGroup.get('queue').patchValue(value, {emitEvent: false}); - this.dirty = true; - } - - onFocus() { - if (this.dirty) { - this.queueFormGroup.get('queue').updateValueAndValidity({onlySelf: true, emitEvent: true}); - this.dirty = false; - } - } - - updateView(value: string | null) { - if (this.modelValue !== value) { - this.modelValue = value; - this.propagateChange(this.modelValue); - } - } - - displayQueueFn(queue?: string): string | undefined { - return queue ? queue : undefined; - } - - fetchQueues(searchText?: string): Observable> { - this.searchText = searchText; - return this.getQueues().pipe( - map(queues => { - const result = queues.filter( queue => { - return searchText ? queue.toUpperCase().startsWith(searchText.toUpperCase()) : true; - }); - if (result.length) { - if (searchText && searchText.length && result.indexOf(searchText) === -1) { - result.push(searchText); - } - result.sort(); - } else if (searchText && searchText.length) { - result.push(searchText); - } - return result; - }) - ); - } - - getQueues(): Observable> { - if (!this.queues) { - this.queues = this.queueService. - getTenantQueuesByServiceType(this.queueType, {ignoreLoading: true}).pipe( - publishReplay(1), - refCount() - ); - } - return this.queues; - } - - clear() { - this.queueFormGroup.get('queue').patchValue(null, {emitEvent: true}); - setTimeout(() => { - this.queueInput.nativeElement.blur(); - this.queueInput.nativeElement.focus(); - }, 0); - } - -} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index cbc030a4fa..856c994943 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -36,7 +36,6 @@ export * from './error.models'; export * from './event.models'; export * from './login.models'; export * from './material.models'; -export * from './queue.models'; export * from './relation.models'; export * from './rule-chain.models'; export * from './rule-node.models'; diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts deleted file mode 100644 index ff8f386dbe..0000000000 --- a/ui-ngx/src/app/shared/models/queue.models.ts +++ /dev/null @@ -1,22 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -export enum ServiceType { - TB_CORE = 'TB_CORE', - TB_RULE_ENGINE = 'TB_RULE_ENGINE', - TB_TRANSPORT = 'TB_TRANSPORT', - JS_EXECUTOR = 'JS_EXECUTOR' -} diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 4f2e01ea81..266981db1f 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -21,7 +21,5 @@ import {TenantId} from './id/tenant-id'; export interface Tenant extends ContactBased { title: string; region: string; - isolatedTbCore: boolean; - isolatedTbRuleEngine: boolean; additionalInfo?: any; } diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index e94c8a6641..5008f10459 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -129,7 +129,6 @@ import { JsonObjectEditDialogComponent } from '@shared/components/dialog/json-ob import { HistorySelectorComponent } from './components/time/history-selector/history-selector.component'; import { TbTemplatePipe } from '@shared/pipe/template.pipe'; import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-gateway-select.component'; -import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list.component'; @NgModule({ providers: [ @@ -180,7 +179,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, - QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, @@ -299,7 +297,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, - QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2a179b3798..3112cba1af 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1514,11 +1514,6 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, - "queue": { - "select_name": "Select queue name", - "name": "Queue Name", - "name_required": "Queue name is required!" - }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1546,12 +1541,7 @@ "no-tenants-matching": "No tenants matching '{{entity}}' were found.", "tenant-required": "Tenant is required", "search": "Search tenants", - "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected", - "tenant-required": "Tenant is required", - "isolated-tb-core": "Processing in isolated ThingsBoard Core container", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json index 0d6c9abdc9..514cd9a8e3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json +++ b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json @@ -1404,11 +1404,6 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, - "queue": { - "select_name": "Выберите имя для Queue", - "name": "Имя для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json index fa0da05e0a..92e8dc4a99 100644 --- a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json +++ b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json @@ -1970,11 +1970,6 @@ "no-timezones-matching": "Не знайдено жодних часових поясів, які відповідають '{{timezone}}'.", "timezone-required": "Необхідно вказати часовий пояс." }, - "queue": { - "select_name": "Виберіть ім'я для Queue", - "name": "Iм'я для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - }, "tenant": { "tenant": "Власник", "tenants": "Власники", diff --git a/ui/package-lock.json b/ui/package-lock.json index bd0e0cc260..6a9997cbf0 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" + "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", + "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" + "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", + "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", + "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", + "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", + "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", + "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", + "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, "attr-accept": { @@ -2102,7 +2102,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2126,7 +2126,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2135,7 +2135,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2144,7 +2144,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2290,7 +2290,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2514,7 +2514,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2545,7 +2545,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2722,7 +2722,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3049,7 +3049,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3109,7 +3109,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", "dev": true }, "convert-source-map": { @@ -3136,7 +3136,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", + "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3355,7 +3355,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", + "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3483,7 +3483,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, "requires": { "ms": "2.0.0" @@ -3561,7 +3561,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3571,7 +3571,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3580,7 +3580,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3589,7 +3589,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3652,7 +3652,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" }, "delegates": { "version": "1.0.0", @@ -3808,7 +3808,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", "dev": true }, "domelementtype": { @@ -3972,7 +3972,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", "dev": true, "requires": { "prr": "~1.0.1" @@ -4491,7 +4491,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4706,7 +4706,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", + "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5134,7 +5134,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", + "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", "dev": true }, "fs-write-stream-atomic": { @@ -5175,7 +5175,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -5196,12 +5197,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5216,17 +5219,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -5343,7 +5349,8 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -5355,6 +5362,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5369,6 +5377,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5376,12 +5385,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5400,6 +5411,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -5489,7 +5501,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -5501,6 +5514,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5586,7 +5600,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5622,6 +5637,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5641,6 +5657,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5684,12 +5701,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -5708,7 +5727,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, "functional-red-black-tree": { @@ -6662,7 +6681,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6712,7 +6731,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6814,7 +6833,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", "dev": true }, "is-callable": { @@ -6858,7 +6877,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6869,7 +6888,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", "dev": true } } @@ -6977,7 +6996,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7044,7 +7063,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", "dev": true }, "is-word-character": { @@ -7184,7 +7203,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, "json-schema": { @@ -7201,7 +7220,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7441,7 +7460,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7974,7 +7993,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" + "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" }, "methods": { "version": "1.1.2", @@ -8016,7 +8035,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", "dev": true }, "mime-db": { @@ -8037,7 +8056,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, "min-document": { "version": "2.19.0", @@ -8094,7 +8113,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { "brace-expansion": "^1.1.7" } @@ -8189,7 +8208,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8267,7 +8286,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8425,7 +8444,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", + "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8485,7 +8504,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", + "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8504,7 +8523,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8705,7 +8724,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9022,7 +9041,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9450,7 +9469,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", + "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9694,7 +9713,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", + "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", "dev": true }, "process": { @@ -9717,7 +9736,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", "requires": { "asap": "~2.0.3" } @@ -9797,7 +9816,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -9962,7 +9981,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", + "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -9993,7 +10012,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", + "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10152,7 +10171,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", + "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10164,7 +10183,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", + "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", "requires": { "lodash": "^4.0.1" } @@ -10347,7 +10366,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" }, "regenerator-transform": { "version": "0.14.1", @@ -10361,7 +10380,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10666,7 +10685,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, "retry": { @@ -10770,7 +10789,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass-graph": { "version": "2.2.4", @@ -11104,7 +11123,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", "dev": true, "requires": { "base": "^0.11.1", @@ -11140,7 +11159,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11160,7 +11179,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11169,7 +11188,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11178,7 +11197,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11191,7 +11210,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11211,7 +11230,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", + "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11335,7 +11354,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11429,7 +11448,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11569,7 +11588,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11613,7 +11632,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" } @@ -12298,7 +12317,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" }, "table": { "version": "5.4.6", @@ -12528,7 +12547,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", "requires": { "os-tmpdir": "~1.0.2" } @@ -12568,7 +12587,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", "dev": true, "requires": { "define-property": "^2.0.2", @@ -12770,7 +12789,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true } } @@ -13095,7 +13114,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13157,7 +13176,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", "dev": true }, "util": { @@ -13922,7 +13941,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", + "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", "dev": true }, "whatwg-fetch": { @@ -14036,7 +14055,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", + "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js deleted file mode 100644 index 0fe93e5fa6..0000000000 --- a/ui/src/app/api/queue.service.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export default angular.module('thingsboard.api.queue', []) - .factory('queueService', queueService) - .name; - -/*@ngInject*/ -function queueService($http, $q) { - var service = { - getTenantQueuesByServiceType: getTenantQueuesByServiceType - }; - - return service; - - function getTenantQueuesByServiceType(serviceType, config) { - let deferred = $q.defer(); - let url = '/api/tenant/queues?serviceType=' + serviceType; - - $http.get(url, config).then(function success(data) { - deferred.resolve(data); - }, function fail() { - deferred.reject(); - }); - return deferred.promise; - } -} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js deleted file mode 100644 index 5236603270..0000000000 --- a/ui/src/app/components/queue/index.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import thingsboardApiQueue from '../../api/queue.service'; -import queueTypeList from "./queue-type-list.directive"; - -export default angular.module('thingsboard.queue', [ - thingsboardApiQueue -]) - .directive('tbQueueTypeList', queueTypeList) - .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js deleted file mode 100644 index 6a2f99d280..0000000000 --- a/ui/src/app/components/queue/queue-type-list.directive.js +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import './queue-type-list.scss'; - -/* eslint-disable import/no-unresolved, import/default */ - -import queueTypeListTemplate from './queue-type-list.tpl.html'; - -/* eslint-enable import/no-unresolved, import/default */ - -/*@ngInject*/ -export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { - - var linker = function (scope, element, attrs, ngModelCtrl) { - var template = $templateCache.get(queueTypeListTemplate); - element.html(template); - - scope.queues = null; - scope.queue = null; - scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; - scope.queueSearchText = ''; - - scope.fetchQueues = function(searchText) { - var deferred = $q.defer(); - loadQueues().then( - function success(queueArr) { - let result = $filter('filter')(queueArr, {'$': searchText}); - if (result && result.length) { - if (searchText && searchText.length && result.indexOf(searchText) === -1) { - result.push(searchText); - } - result.sort(); - deferred.resolve(result); - } else { - deferred.resolve([searchText]); - } - }, - function fail() { - deferred.reject(); - } - ); - - return deferred.promise; - }; - - scope.updateView = function () { - if (!scope.disabled) { - ngModelCtrl.$setViewValue(scope.queue); - } - }; - - function loadQueues() { - var deferred = $q.defer(); - if (!scope.queues) { - queueService.getTenantQueuesByServiceType(scope.queueType).then( - function success(queueArr) { - scope.queues = queueArr.data; - deferred.resolve(scope.queues); - }, - function fail() { - deferred.reject(); - } - ); - } else { - deferred.resolve(scope.queues); - } - return deferred.promise; - } - - ngModelCtrl.$render = function () { - scope.queue = ngModelCtrl.$viewValue; - }; - - scope.$watch('queue', function (newValue, prevValue) { - if (!angular.equals(newValue, prevValue)) { - scope.updateView(); - } - }); - - scope.$watch('disabled', function () { - scope.updateView(); - }); - - $compile(element.contents())(scope); - }; - - return { - restrict: "E", - require: "^ngModel", - link: linker, - scope: { - theForm: '=?', - tbRequired: '=?', - disabled:'=ngDisabled', - queueType: '=?' - } - }; -} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss deleted file mode 100644 index b8f6f58fb2..0000000000 --- a/ui/src/app/components/queue/queue-type-list.scss +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html deleted file mode 100644 index 5fc700026e..0000000000 --- a/ui/src/app/components/queue/queue-type-list.tpl.html +++ /dev/null @@ -1,43 +0,0 @@ - - - -
- {{item}} -
- -
-
{{'queue.name_required' | translate}}
-
- \ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index f3b4d10e4f..a00baa5199 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,7 +55,6 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; -import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -87,7 +86,6 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, - thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index fa4689fc75..8c7a89580d 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1489,12 +1489,6 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, - "queue": { - "select_name": "Select queue name", - "name": "Queue Name", - "name_required": "Queue name is required!" - - }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1520,11 +1514,7 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required", - "isolated-tb-core": "Processing in isolated ThingsBoard Core container", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "tenant-required": "Tenant is required" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index e93c92124c..f86eb988ff 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,12 +1403,6 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, - "queue": { - "select_name": "Выберите имя для Queue", - "name": "Имя для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - - }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index c19049bc11..5a0899014d 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,12 +1820,6 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, - "queue": { - "select_name": "Виберіть ім'я для Queue", - "name": "Iм'я для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - - }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index b75ea30f06..aa7751d479 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,50 +15,32 @@ limitations under the License. --> -{{ - 'tenant.manage-tenant-admins' | translate }} - -{{ 'tenant.delete' - | translate }} - +{{ 'tenant.manage-tenant-admins' | translate }} +{{ 'tenant.delete' | translate }}
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
- - - - - - - - - {{'tenant.isolated-tb-core' | translate}}
- {{'tenant.isolated-tb-core-details' | translate}} - - - - - {{'tenant.isolated-tb-rule-engine' | translate}}
- {{'tenant.isolated-tb-rule-engine-details' | translate}} - - -
+
+ + + +
+
tenant.title-required
+
+ + + + + + +
From df3614f9e4d26ce6cd6479e24402b17829a82d95 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Apr 2020 01:12:23 +0300 Subject: [PATCH 095/102] Fix Alarm repository. Add queue type list select. --- .../CassandraEntitiesToSqlMigrateService.java | 7 +- .../migrate/CassandraToSqlEventTsColumn.java | 40 ++++ .../thingsboard/server/dao/util/SqlDao.java | 3 - .../server/dao/sql/alarm/AlarmRepository.java | 18 +- ui-ngx/src/app/core/http/public-api.ts | 1 + ui-ngx/src/app/core/http/queue.service.ts | 36 ++++ .../home/pages/tenant/tenant.component.html | 10 + .../home/pages/tenant/tenant.component.scss | 22 +++ .../home/pages/tenant/tenant.component.ts | 7 +- .../queue/queue-type-list.component.html | 43 +++++ .../queue/queue-type-list.component.ts | 177 ++++++++++++++++++ ui-ngx/src/app/shared/models/public-api.ts | 1 + ui-ngx/src/app/shared/models/queue.models.ts | 22 +++ ui-ngx/src/app/shared/models/tenant.model.ts | 2 + ui-ngx/src/app/shared/shared.module.ts | 3 + .../assets/locale/locale.constant-en_US.json | 11 +- .../assets/locale/locale.constant-ru_RU.json | 5 + .../assets/locale/locale.constant-uk_UA.json | 5 + 18 files changed, 404 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java create mode 100644 ui-ngx/src/app/core/http/queue.service.ts create mode 100644 ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss create mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.html create mode 100644 ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts create mode 100644 ui-ngx/src/app/shared/models/queue.models.ts diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index 08a5fdd3e9..e7f21a18d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -203,7 +203,8 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("entity_type"), stringColumn("event_type"), stringColumn("event_uid"), - stringColumn("body")), + stringColumn("body"), + new CassandraToSqlEventTsColumn()), new CassandraToSqlTable("relation", idColumn("from_id"), stringColumn("from_type"), @@ -245,7 +246,9 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("zip"), stringColumn("phone"), stringColumn("email"), - stringColumn("additional_info")), + stringColumn("additional_info"), + booleanColumn("isolated_tb_core"), + booleanColumn("isolated_tb_rule_engine")), new CassandraToSqlTable("user_credentials", idColumn("id"), idColumn("user_id"), diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java new file mode 100644 index 0000000000..b488e812fe --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.migrate; + +import com.datastax.driver.core.Row; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; + +public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { + + CassandraToSqlEventTsColumn() { + super("id", "ts", CassandraToSqlColumnType.BIGINT, null); + } + + @Override + public String getColumnValue(Row row) { + UUID id = row.getUUID(getIndex()); + long ts = getTs(id); + return ts + ""; + } + + private long getTs(UUID uuid) { + return (uuid.timestamp() - EPOCH_DIFF) / 10000; + } +} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 408f81a845..76db7920a6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -15,12 +15,9 @@ */ package org.thingsboard.server.dao.util; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; - import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql") public @interface SqlDao { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 2f3c72d01b..78230332b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -38,7 +38,7 @@ public interface AlarmRepository extends CrudRepository { @Param("alarmType") String alarmType, Pageable pageable); - @Query("SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + + @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + "RelationEntity re " + "WHERE a.tenantId = :tenantId " + "AND a.id = re.toId AND re.toType = 'ALARM' " + @@ -51,7 +51,21 @@ public interface AlarmRepository extends CrudRepository { "AND (:idOffset IS NULL OR a.id < :idOffset) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))", + countQuery = "SELECT count(a) FROM AlarmEntity a, " + + "RelationEntity re " + + "WHERE a.tenantId = :tenantId " + + "AND a.id = re.toId AND re.toType = 'ALARM' " + + "AND re.relationTypeGroup = 'ALARM' " + + "AND re.relationType = :relationType " + + "AND re.fromId = :affectedEntityId " + + "AND re.fromType = :affectedEntityType " + + "AND (:startId IS NULL OR a.id >= :startId) " + + "AND (:endId IS NULL OR a.id <= :endId) " + + "AND (:idOffset IS NULL OR a.id < :idOffset) " + + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") Page findAlarms(@Param("tenantId") String tenantId, @Param("affectedEntityId") String affectedEntityId, @Param("affectedEntityType") String affectedEntityType, diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index 761f04989d..db477c3a8c 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -28,6 +28,7 @@ export * from './entity-relation.service'; export * from './entity-view.service'; export * from './event.service'; export * from './http-utils'; +export * from './queue.service'; export * from './rule-chain.service'; export * from './tenant.service'; export * from './user.service'; diff --git a/ui-ngx/src/app/core/http/queue.service.ts b/ui-ngx/src/app/core/http/queue.service.ts new file mode 100644 index 0000000000..42beadb81a --- /dev/null +++ b/ui-ngx/src/app/core/http/queue.service.ts @@ -0,0 +1,36 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable } from 'rxjs'; +import { ServiceType } from '@shared/models/queue.models'; + +@Injectable({ + providedIn: 'root' +}) +export class QueueService { + + constructor( + private http: HttpClient + ) { } + + public getTenantQueuesByServiceType(serviceType: ServiceType, config?: RequestConfig): Observable> { + return this.http.get>(`/api/tenant/queues?serviceType=${serviceType}`, + defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html index 48c57184ab..7d0ccd5056 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html @@ -56,6 +56,16 @@ +
+ +
{{ 'tenant.isolated-tb-core' | translate }}
+
{{'tenant.isolated-tb-core-details' | translate}}
+ + +
{{ 'tenant.isolated-tb-rule-engine' | translate }}
+
{{'tenant.isolated-tb-rule-engine-details' | translate}}
+ +
diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss new file mode 100644 index 0000000000..b05c6324fa --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .mat-checkbox.hinted-checkbox { + .mat-checkbox-inner-container { + margin-top: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts index c8317a20c5..f624a5d5cf 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts @@ -26,7 +26,8 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod @Component({ selector: 'tb-tenant', - templateUrl: './tenant.component.html' + templateUrl: './tenant.component.html', + styleUrls: ['./tenant.component.scss'] }) export class TenantComponent extends ContactBasedComponent { @@ -50,6 +51,8 @@ export class TenantComponent extends ContactBasedComponent { return this.fb.group( { title: [entity ? entity.title : '', [Validators.required]], + isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], + isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], additionalInfo: this.fb.group( { description: [entity && entity.additionalInfo ? entity.additionalInfo.description : ''] @@ -61,6 +64,8 @@ export class TenantComponent extends ContactBasedComponent { updateEntityForm(entity: Tenant) { this.entityForm.patchValue({title: entity.title}); + this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); + this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); this.entityForm.patchValue({additionalInfo: {description: entity.additionalInfo ? entity.additionalInfo.description : ''}}); } diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html new file mode 100644 index 0000000000..ccdf4a76ec --- /dev/null +++ b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html @@ -0,0 +1,43 @@ + + + {{ 'queue.name' | translate }} + + + + + + + + + {{ 'queue.name_required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts new file mode 100644 index 0000000000..7e65bfbd91 --- /dev/null +++ b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts @@ -0,0 +1,177 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable } from 'rxjs'; +import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { QueueService } from '@core/http/queue.service'; +import { ServiceType } from '@shared/models/queue.models'; + +@Component({ + selector: 'tb-queue-type-list', + templateUrl: './queue-type-list.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => QueueTypeListComponent), + multi: true + }] +}) +export class QueueTypeListComponent implements ControlValueAccessor, OnInit, AfterViewInit, OnDestroy { + + queueFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + @Input() + queueType: ServiceType; + + @ViewChild('queueInput', {static: true}) queueInput: ElementRef; + + filteredQueues: Observable>; + + queues: Observable>; + + searchText = ''; + + private dirty = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + public translate: TranslateService, + private queueService: QueueService, + private fb: FormBuilder) { + this.queueFormGroup = this.fb.group({ + queue: [null] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.filteredQueues = this.queueFormGroup.get('queue').valueChanges + .pipe( + tap(value => { + this.updateView(value); + }), + map(value => value ? value : ''), + mergeMap(queue => this.fetchQueues(queue) ) + ); + } + + ngAfterViewInit(): void { + } + + ngOnDestroy(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.queueFormGroup.disable({emitEvent: false}); + } else { + this.queueFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + this.queueFormGroup.get('queue').patchValue(value, {emitEvent: false}); + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.queueFormGroup.get('queue').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + updateView(value: string | null) { + if (this.modelValue !== value) { + this.modelValue = value; + this.propagateChange(this.modelValue); + } + } + + displayQueueFn(queue?: string): string | undefined { + return queue ? queue : undefined; + } + + fetchQueues(searchText?: string): Observable> { + this.searchText = searchText; + return this.getQueues().pipe( + map(queues => { + const result = queues.filter( queue => { + return searchText ? queue.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + } else if (searchText && searchText.length) { + result.push(searchText); + } + return result; + }) + ); + } + + getQueues(): Observable> { + if (!this.queues) { + this.queues = this.queueService. + getTenantQueuesByServiceType(this.queueType, {ignoreLoading: true}).pipe( + publishReplay(1), + refCount() + ); + } + return this.queues; + } + + clear() { + this.queueFormGroup.get('queue').patchValue(null, {emitEvent: true}); + setTimeout(() => { + this.queueInput.nativeElement.blur(); + this.queueInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 856c994943..cbc030a4fa 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -36,6 +36,7 @@ export * from './error.models'; export * from './event.models'; export * from './login.models'; export * from './material.models'; +export * from './queue.models'; export * from './relation.models'; export * from './rule-chain.models'; export * from './rule-node.models'; diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts new file mode 100644 index 0000000000..ff8f386dbe --- /dev/null +++ b/ui-ngx/src/app/shared/models/queue.models.ts @@ -0,0 +1,22 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export enum ServiceType { + TB_CORE = 'TB_CORE', + TB_RULE_ENGINE = 'TB_RULE_ENGINE', + TB_TRANSPORT = 'TB_TRANSPORT', + JS_EXECUTOR = 'JS_EXECUTOR' +} diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 266981db1f..4f2e01ea81 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -21,5 +21,7 @@ import {TenantId} from './id/tenant-id'; export interface Tenant extends ContactBased { title: string; region: string; + isolatedTbCore: boolean; + isolatedTbRuleEngine: boolean; additionalInfo?: any; } diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 5008f10459..e94c8a6641 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -129,6 +129,7 @@ import { JsonObjectEditDialogComponent } from '@shared/components/dialog/json-ob import { HistorySelectorComponent } from './components/time/history-selector/history-selector.component'; import { TbTemplatePipe } from '@shared/pipe/template.pipe'; import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-gateway-select.component'; +import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list.component'; @NgModule({ providers: [ @@ -179,6 +180,7 @@ import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-g EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, + QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, @@ -297,6 +299,7 @@ import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-g EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, + QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3112cba1af..4ef7b845c7 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1514,6 +1514,11 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1541,7 +1546,11 @@ "no-tenants-matching": "No tenants matching '{{entity}}' were found.", "tenant-required": "Tenant is required", "search": "Search tenants", - "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected" + "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json index 514cd9a8e3..0d6c9abdc9 100644 --- a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json +++ b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json @@ -1404,6 +1404,11 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json index 92e8dc4a99..fa0da05e0a 100644 --- a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json +++ b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json @@ -1970,6 +1970,11 @@ "no-timezones-matching": "Не знайдено жодних часових поясів, які відповідають '{{timezone}}'.", "timezone-required": "Необхідно вказати часовий пояс." }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + }, "tenant": { "tenant": "Власник", "tenants": "Власники", From e175b571100992f2b864538822519513c73b3926 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Mon, 27 Apr 2020 09:26:17 +0300 Subject: [PATCH 096/102] [3.0] Entity list not correct find index for model (#2668) * Entity list not correct find index for model * Clear import * Revert change * Revert change --- .../src/app/shared/components/entity/entity-list.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts index 91aac0703b..f8bdab6243 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts @@ -205,10 +205,11 @@ export class EntityListComponent implements ControlValueAccessor, OnInit, AfterV } remove(entity: BaseData) { - const index = this.entities.indexOf(entity); + let index = this.entities.indexOf(entity); if (index >= 0) { this.entities.splice(index, 1); this.entityListFormGroup.get('entities').setValue(this.entities); + index = this.modelValue.indexOf(entity.id.id); this.modelValue.splice(index, 1); if (!this.modelValue.length) { this.modelValue = null; From e3e860fe3f67cc2a30b5ac01e772220854772587 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Mon, 27 Apr 2020 09:27:14 +0300 Subject: [PATCH 097/102] [3.0] Improvement code style (#2670) * Clear code and styles * Clear import --- ui-ngx/src/app/app-routing.module.ts | 2 +- ui-ngx/src/app/core/api/data-aggregator.ts | 4 +- .../app/core/api/datasource-subcription.ts | 2 +- ui-ngx/src/app/core/api/widget-api.models.ts | 2 +- ui-ngx/src/app/core/core.module.ts | 3 +- ui-ngx/src/app/core/core.state.ts | 8 +--- ui-ngx/src/app/core/guards/auth.guard.ts | 7 +--- .../app/core/guards/confirm-on-exit.guard.ts | 6 +-- ui-ngx/src/app/core/http/alarm.service.ts | 5 +-- .../core/http/component-descriptor.service.ts | 3 +- ui-ngx/src/app/core/http/device.service.ts | 3 +- ui-ngx/src/app/core/http/entity.service.ts | 9 ---- .../app/core/interceptors/load.selectors.ts | 2 - .../core/notification/notification.actions.ts | 2 +- .../core/notification/notification.effects.ts | 9 +--- .../app/core/services/broadcast.service.ts | 9 ++-- .../core/services/dashboard-utils.service.ts | 1 - .../app/core/services/item-buffer.service.ts | 2 +- ui-ngx/src/app/core/services/menu.service.ts | 2 +- .../app/core/services/notification.service.ts | 2 +- ui-ngx/src/app/core/services/raf.service.ts | 1 - .../script/node-script-test.service.ts | 9 ++-- ui-ngx/src/app/core/services/time.service.ts | 9 +++- .../src/app/core/services/window.service.ts | 2 +- .../src/app/core/settings/settings.effects.ts | 19 ++------- .../translate/missing-translate-handler.ts | 2 +- .../translate/translate-default-compiler.ts | 4 +- ui-ngx/src/app/core/utils.ts | 4 +- .../dashboard-pages.routing.module.ts | 10 +---- .../dashboard/dashboard-routing.module.ts | 2 +- ...-widget-to-dashboard-dialog.component.html | 2 +- .../entity/contact-based.component.ts | 2 +- .../entity/entities-table.component.ts | 12 ++++-- .../entity/entity-details-panel.component.ts | 9 ++-- .../entity/entity-filter.component.html | 4 +- .../entity/entity-table-header.component.ts | 3 +- .../entity/entity-tabs.component.ts | 6 +-- .../import-export/import-dialog.component.ts | 19 +-------- ...ction-pretty-resources-tabs.component.scss | 2 +- .../widget/data-keys.component.scss | 2 +- .../widget/dialog/custom-dialog.service.ts | 3 +- .../widget/legend-config-panel.component.ts | 3 +- .../lib/alarms-table-widget.component.ts | 6 +-- .../widget/lib/analogue-radial-gauge.ts | 3 +- .../date-range-navigator.component.html | 6 +-- .../date-range-navigator.component.scss | 10 ----- .../date-range-navigator.component.ts | 2 +- .../lib/entities-table-widget.component.ts | 32 +++++++-------- .../widget/lib/flot-widget.models.ts | 2 +- .../components/widget/lib/maps/leaflet-map.ts | 13 ++++-- .../components/widget/lib/maps/map-widget2.ts | 30 +++++++------- .../components/widget/lib/maps/maps-utils.ts | 6 +-- .../components/widget/lib/maps/markers.scss | 4 +- .../widget/lib/maps/providers/google-map.ts | 4 +- .../widget/lib/maps/providers/here-map.ts | 4 +- .../widget/lib/maps/providers/image-map.ts | 4 +- .../lib/maps/providers/openstreet-map.ts | 4 +- .../widget/lib/maps/providers/tencent-map.ts | 4 +- .../widget/lib/rpc/knob.component.ts | 2 +- .../components/widget/lib/table-widget.scss | 6 +-- .../lib/timeseries-table-widget.component.ts | 3 +- .../trip-animation.component.ts | 8 ++-- .../widget/widget-config.component.ts | 4 +- ...d-entities-to-customer-dialog.component.ts | 4 -- .../assign-to-customer-dialog.component.ts | 3 -- .../app/modules/home/home-routing.module.ts | 2 +- .../modules/home/menu/menu-link.component.ts | 2 +- .../home/menu/menu-toggle.component.ts | 2 +- .../home/models/dashboard-component.models.ts | 4 +- .../models/datasource/attribute-datasource.ts | 7 ++-- .../models/entity/entity-component.models.ts | 4 +- .../home/models/widget-component.models.ts | 22 +++++----- .../dashboard/dashboard-page.component.ts | 6 ++- .../pages/dashboard/dashboard-page.models.ts | 1 - .../dashboard-widget-select.component.ts | 15 +------ .../pages/dashboard/edit-widget.component.ts | 25 ++--------- .../make-dashboard-public-dialog.component.ts | 20 ++++----- ...ge-dashboard-customers-dialog.component.ts | 18 ++++---- .../states/state-controller.component.ts | 2 +- .../device/device-table-header.component.ts | 4 +- .../home-links/home-links-routing.module.ts | 8 ++-- .../pages/home-links/home-links.component.ts | 3 -- .../pages/home-links/home-links.module.ts | 2 +- .../rulechain/rulechain-page.component.scss | 2 +- .../rulechain/rulechain-page.component.ts | 8 ++-- .../rulechain/rulechain-routing.module.ts | 10 +++-- .../rulechain/rulechain-tabs.component.ts | 1 - .../pages/widget/widget-editor.component.html | 2 +- .../pages/widget/widget-editor.component.ts | 2 +- .../pages/widget/widget-library.module.ts | 12 +++--- .../pages/widget/widgets-bundle.component.ts | 10 ++--- .../login/pages/login/login.component.ts | 3 +- .../animations/speed-dial-fab.animations.ts | 11 +---- .../components/circular-progress.directive.ts | 4 +- .../material-icons-dialog.component.scss | 2 +- .../node-script-test-dialog.component.html | 2 +- .../dialog/todo-dialog.component.ts | 2 +- .../directives/tb-json-to-string.directive.ts | 10 +++-- .../entity/entity-list-select.component.ts | 18 ++++---- .../entity/entity-list.component.ts | 39 +++++++----------- .../entity-subtype-autocomplete.component.ts | 34 ++++++--------- .../entity/entity-subtype-list.component.ts | 41 ++++--------------- .../components/fab-toolbar.component.ts | 1 - .../components/file-input.component.scss | 2 +- .../shared/components/file-input.component.ts | 3 +- .../footer-fab-buttons.component.scss | 2 +- .../shared/components/fullscreen.directive.ts | 9 +++- .../shared/components/hotkeys.directive.ts | 4 +- .../components/image-input.component.scss | 2 +- .../json-form/react/json-form-date.tsx | 5 +-- .../json-form/react/json-form-image.tsx | 5 +-- .../json-form/react/json-form-radios.tsx | 2 +- .../json-form/react/json-form-rc-select.tsx | 2 +- .../json-form/react/json-form-schema-form.tsx | 3 +- .../json-form/react/json-form-select.tsx | 6 +-- .../json-form/react/json-form-text.tsx | 1 - .../json-form/react/json-form-utils.ts | 4 +- .../components/json-form/react/json-form.scss | 2 +- .../app/shared/components/kv-map.component.ts | 20 ++++----- .../shared/components/led-light.component.ts | 3 +- .../shared/components/nav-tree.component.scss | 1 - .../components/snack-bar-component.scss | 8 ++-- .../shared/components/tb-error.component.ts | 2 +- .../history-selector.component.scss | 10 ----- .../history-selector.component.ts | 2 +- .../app/shared/components/toast.directive.ts | 12 ++++-- .../components/value-input.component.html | 2 +- .../widgets-bundle-select.component.ts | 2 +- ui-ngx/src/app/shared/models/alias.models.ts | 2 +- ui-ngx/src/app/shared/models/asset.models.ts | 9 ++-- .../src/app/shared/models/customer.model.ts | 2 +- .../src/app/shared/models/dashboard.models.ts | 3 +- ui-ngx/src/app/shared/models/device.models.ts | 10 ++--- .../app/shared/models/entity-type.models.ts | 2 +- .../app/shared/models/entity-view.models.ts | 11 +++-- ui-ngx/src/app/shared/models/id/entity-id.ts | 2 +- .../src/app/shared/models/material.models.ts | 1 + .../src/app/shared/models/page/page-data.ts | 3 -- .../src/app/shared/models/relation.models.ts | 1 - ui-ngx/src/app/shared/models/tenant.model.ts | 3 +- ui-ngx/src/app/shared/models/user.model.ts | 2 +- .../app/shared/models/widgets-bundle.model.ts | 6 +-- ui-ngx/src/app/shared/pipe/highlight.pipe.ts | 2 +- ui-ngx/src/app/shared/pipe/tbJson.pipe.ts | 4 +- ui-ngx/src/app/shared/pipe/template.pipe.ts | 2 +- ui-ngx/src/app/shared/pipe/truncate.pipe.ts | 2 +- ui-ngx/src/app/shared/shared.module.ts | 8 +++- ui-ngx/src/styles.scss | 3 +- 148 files changed, 382 insertions(+), 547 deletions(-) diff --git a/ui-ngx/src/app/app-routing.module.ts b/ui-ngx/src/app/app-routing.module.ts index e0f573095d..3e784284d4 100644 --- a/ui-ngx/src/app/app-routing.module.ts +++ b/ui-ngx/src/app/app-routing.module.ts @@ -15,7 +15,7 @@ /// import { NgModule } from '@angular/core'; -import { Routes, RouterModule } from '@angular/router'; +import { RouterModule, Routes } from '@angular/router'; const routes: Routes = [ { path: '', diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index a5a1b65bf0..69ffecbdaf 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { SubscriptionData, SubscriptionUpdateMsg, SubscriptionDataHolder } from '@app/shared/models/telemetry/telemetry.models'; +import { SubscriptionData, SubscriptionDataHolder } from '@app/shared/models/telemetry/telemetry.models'; import { AggregationType } from '@shared/models/time/time.models'; import { UtilsService } from '@core/services/utils.service'; -import Timeout = NodeJS.Timeout; import { deepClone } from '@core/utils'; +import Timeout = NodeJS.Timeout; export declare type onAggregatedData = (data: SubscriptionData, detectChanges: boolean) => void; diff --git a/ui-ngx/src/app/core/api/datasource-subcription.ts b/ui-ngx/src/app/core/api/datasource-subcription.ts index 0c4edfd349..dc118b4491 100644 --- a/ui-ngx/src/app/core/api/datasource-subcription.ts +++ b/ui-ngx/src/app/core/api/datasource-subcription.ts @@ -28,7 +28,7 @@ import { } from '@shared/models/telemetry/telemetry.models'; import { DatasourceListener } from './datasource.service'; import { AggregationType, SubscriptionTimewindow, YEAR } from '@shared/models/time/time.models'; -import { deepClone, isDefined, isObject, isDefinedAndNotNull } from '@core/utils'; +import { deepClone, isDefinedAndNotNull, isObject } from '@core/utils'; import { UtilsService } from '@core/services/utils.service'; import { EntityType } from '@shared/models/entity-type.models'; import { DataAggregator } from '@core/api/data-aggregator'; diff --git a/ui-ngx/src/app/core/api/widget-api.models.ts b/ui-ngx/src/app/core/api/widget-api.models.ts index 0eeb2af080..9a7d855d6e 100644 --- a/ui-ngx/src/app/core/api/widget-api.models.ts +++ b/ui-ngx/src/app/core/api/widget-api.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Observable, Subject } from 'rxjs'; +import { Observable } from 'rxjs'; import { EntityId } from '@app/shared/models/id/entity-id'; import { DataSet, diff --git a/ui-ngx/src/app/core/core.module.ts b/ui-ngx/src/app/core/core.module.ts index b053938722..3defaea872 100644 --- a/ui-ngx/src/app/core/core.module.ts +++ b/ui-ngx/src/app/core/core.module.ts @@ -28,7 +28,8 @@ import { MissingTranslationHandler, TranslateCompiler, TranslateLoader, - TranslateModule, TranslateParser + TranslateModule, + TranslateParser } from '@ngx-translate/core'; import { TranslateHttpLoader } from '@ngx-translate/http-loader'; import { TbMissingTranslationHandler } from './translate/missing-translate-handler'; diff --git a/ui-ngx/src/app/core/core.state.ts b/ui-ngx/src/app/core/core.state.ts index 45ed3ce4c1..6dc03f3b0a 100644 --- a/ui-ngx/src/app/core/core.state.ts +++ b/ui-ngx/src/app/core/core.state.ts @@ -14,13 +14,10 @@ /// limitations under the License. /// -import { - ActionReducerMap, - MetaReducer, Store -} from '@ngrx/store'; +import { ActionReducerMap, MetaReducer } from '@ngrx/store'; import { storeFreeze } from 'ngrx-store-freeze'; -import { environment as env} from '@env/environment'; +import { environment as env } from '@env/environment'; import { initStateFromLocalStorage } from './meta-reducers/init-state-from-local-storage.reducer'; import { debug } from './meta-reducers/debug.reducer'; @@ -35,7 +32,6 @@ import { SettingsEffects } from '@app/core/settings/settings.effects'; import { NotificationState } from '@app/core/notification/notification.models'; import { notificationReducer } from '@app/core/notification/notification.reducer'; import { NotificationEffects } from '@app/core/notification/notification.effects'; -import { take } from 'rxjs/operators'; export const reducers: ActionReducerMap = { load: loadReducer, diff --git a/ui-ngx/src/app/core/guards/auth.guard.ts b/ui-ngx/src/app/core/guards/auth.guard.ts index f42029f6f9..98a0e781d2 100644 --- a/ui-ngx/src/app/core/guards/auth.guard.ts +++ b/ui-ngx/src/app/core/guards/auth.guard.ts @@ -15,12 +15,7 @@ /// import { Injectable, NgZone } from '@angular/core'; -import { - ActivatedRouteSnapshot, - CanActivate, - CanActivateChild, - RouterStateSnapshot -} from '@angular/router'; +import { ActivatedRouteSnapshot, CanActivate, CanActivateChild, RouterStateSnapshot } from '@angular/router'; import { AuthService } from '../auth/auth.service'; import { select, Store } from '@ngrx/store'; import { AppState } from '../core.state'; diff --git a/ui-ngx/src/app/core/guards/confirm-on-exit.guard.ts b/ui-ngx/src/app/core/guards/confirm-on-exit.guard.ts index 6a76f27056..e3ee9a5330 100644 --- a/ui-ngx/src/app/core/guards/confirm-on-exit.guard.ts +++ b/ui-ngx/src/app/core/guards/confirm-on-exit.guard.ts @@ -15,11 +15,7 @@ /// import { Injectable } from '@angular/core'; -import { - ActivatedRouteSnapshot, - CanDeactivate, - RouterStateSnapshot -} from '@angular/router'; +import { ActivatedRouteSnapshot, CanDeactivate, RouterStateSnapshot } from '@angular/router'; import { FormGroup } from '@angular/forms'; import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; diff --git a/ui-ngx/src/app/core/http/alarm.service.ts b/ui-ngx/src/app/core/http/alarm.service.ts index 3e2bf58285..417975aebb 100644 --- a/ui-ngx/src/app/core/http/alarm.service.ts +++ b/ui-ngx/src/app/core/http/alarm.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { EMPTY, Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -36,9 +36,8 @@ import { UtilsService } from '@core/services/utils.service'; import { TimePageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { concatMap, expand, map, toArray } from 'rxjs/operators'; -import { EMPTY } from 'rxjs'; -import Timeout = NodeJS.Timeout; import { isDefined } from '@core/utils'; +import Timeout = NodeJS.Timeout; interface AlarmSourceListenerQuery { entityType: EntityType; diff --git a/ui-ngx/src/app/core/http/component-descriptor.service.ts b/ui-ngx/src/app/core/http/component-descriptor.service.ts index 3eb5725ac8..a3198b862a 100644 --- a/ui-ngx/src/app/core/http/component-descriptor.service.ts +++ b/ui-ngx/src/app/core/http/component-descriptor.service.ts @@ -19,8 +19,7 @@ import { HttpClient } from '@angular/common/http'; import { ComponentDescriptor, ComponentType } from '@shared/models/component-descriptor.models'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { Observable, of } from 'rxjs'; -import { RuleChainMetaData } from '@shared/models/rule-chain.models'; -import { tap, map } from 'rxjs/operators'; +import { map } from 'rxjs/operators'; import { RuleNodeType } from '@shared/models/rule-node.models'; @Injectable({ diff --git a/ui-ngx/src/app/core/http/device.service.ts b/ui-ngx/src/app/core/http/device.service.ts index b432c81e41..2d31503bd1 100644 --- a/ui-ngx/src/app/core/http/device.service.ts +++ b/ui-ngx/src/app/core/http/device.service.ts @@ -21,7 +21,8 @@ import { HttpClient } from '@angular/common/http'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { - ClaimRequest, ClaimResult, + ClaimRequest, + ClaimResult, Device, DeviceCredentials, DeviceInfo, diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 4cf67175e4..3eb80ef6a3 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -642,7 +642,6 @@ export class EntityService { return result; } )); - break; case AliasFilterType.entityList: return this.getEntities(filter.entityType, filter.entityList, {ignoreLoading: true, ignoreErrors: true}).pipe( map((entities) => { @@ -654,7 +653,6 @@ export class EntityService { } } )); - break; case AliasFilterType.entityName: return this.getEntitiesByNameFilter(filter.entityType, filter.entityNameFilter, maxItems, '', {ignoreLoading: true, ignoreErrors: true}).pipe( @@ -668,7 +666,6 @@ export class EntityService { } ) ); - break; case AliasFilterType.stateEntity: result.stateEntity = true; if (stateEntityId) { @@ -681,7 +678,6 @@ export class EntityService { } else { return of(result); } - break; case AliasFilterType.assetType: return this.getEntitiesByNameFilter(EntityType.ASSET, filter.assetNameFilter, maxItems, filter.assetType, {ignoreLoading: true, ignoreErrors: true}).pipe( @@ -695,7 +691,6 @@ export class EntityService { } ) ); - break; case AliasFilterType.deviceType: return this.getEntitiesByNameFilter(EntityType.DEVICE, filter.deviceNameFilter, maxItems, filter.deviceType, {ignoreLoading: true, ignoreErrors: true}).pipe( @@ -709,7 +704,6 @@ export class EntityService { } ) ); - break; case AliasFilterType.entityViewType: return this.getEntitiesByNameFilter(EntityType.ENTITY_VIEW, filter.entityViewNameFilter, maxItems, filter.entityViewType, {ignoreLoading: true, ignoreErrors: true}).pipe( @@ -723,7 +717,6 @@ export class EntityService { } ) ); - break; case AliasFilterType.relationsQuery: result.stateEntity = filter.rootStateEntity; let rootEntityType; @@ -768,7 +761,6 @@ export class EntityService { } else { return of(result); } - break; case AliasFilterType.assetSearchQuery: case AliasFilterType.deviceSearchQuery: case AliasFilterType.entityViewSearchQuery: @@ -823,7 +815,6 @@ export class EntityService { } else { return of(result); } - break; } } diff --git a/ui-ngx/src/app/core/interceptors/load.selectors.ts b/ui-ngx/src/app/core/interceptors/load.selectors.ts index 280653361f..a423d6c9ab 100644 --- a/ui-ngx/src/app/core/interceptors/load.selectors.ts +++ b/ui-ngx/src/app/core/interceptors/load.selectors.ts @@ -18,9 +18,7 @@ import { createFeatureSelector, createSelector, select, Store } from '@ngrx/stor import { AppState } from '../core.state'; import { LoadState } from './load.models'; -import { AuthUser } from '@shared/models/user.model'; import { take } from 'rxjs/operators'; -import { selectAuthUser } from '@core/auth/auth.selectors'; export const selectLoadState = createFeatureSelector( 'load' diff --git a/ui-ngx/src/app/core/notification/notification.actions.ts b/ui-ngx/src/app/core/notification/notification.actions.ts index a7cc19fe14..098e5cd2c8 100644 --- a/ui-ngx/src/app/core/notification/notification.actions.ts +++ b/ui-ngx/src/app/core/notification/notification.actions.ts @@ -15,7 +15,7 @@ /// import { Action } from '@ngrx/store'; -import { NotificationMessage, HideNotification } from '@app/core/notification/notification.models'; +import { HideNotification, NotificationMessage } from '@app/core/notification/notification.models'; export enum NotificationActionTypes { SHOW_NOTIFICATION = '[Notification] Show', diff --git a/ui-ngx/src/app/core/notification/notification.effects.ts b/ui-ngx/src/app/core/notification/notification.effects.ts index 017ba5fd77..a7dd698172 100644 --- a/ui-ngx/src/app/core/notification/notification.effects.ts +++ b/ui-ngx/src/app/core/notification/notification.effects.ts @@ -16,14 +16,9 @@ import { Injectable } from '@angular/core'; import { Actions, Effect, ofType } from '@ngrx/effects'; -import { - map, -} from 'rxjs/operators'; +import { map } from 'rxjs/operators'; -import { - NotificationActions, - NotificationActionTypes -} from '@app/core/notification/notification.actions'; +import { NotificationActions, NotificationActionTypes } from '@app/core/notification/notification.actions'; import { NotificationService } from '@app/core/services/notification.service'; @Injectable() diff --git a/ui-ngx/src/app/core/services/broadcast.service.ts b/ui-ngx/src/app/core/services/broadcast.service.ts index 90a8bcff4c..92dc2ef98e 100644 --- a/ui-ngx/src/app/core/services/broadcast.service.ts +++ b/ui-ngx/src/app/core/services/broadcast.service.ts @@ -14,11 +14,10 @@ /// limitations under the License. /// -import {Injectable} from '@angular/core'; -import {Subject, Subscription} from 'rxjs'; -import {NotificationMessage} from '@core/notification/notification.models'; -import {BroadcastEvent, BroadcastListener, BroadcastMessage} from '@core/services/broadcast.models'; -import {filter} from 'rxjs/operators'; +import { Injectable } from '@angular/core'; +import { Subject, Subscription } from 'rxjs'; +import { BroadcastEvent, BroadcastListener, BroadcastMessage } from '@core/services/broadcast.models'; +import { filter } from 'rxjs/operators'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/dashboard-utils.service.ts b/ui-ngx/src/app/core/services/dashboard-utils.service.ts index c892d02c34..15f0b23d6a 100644 --- a/ui-ngx/src/app/core/services/dashboard-utils.service.ts +++ b/ui-ngx/src/app/core/services/dashboard-utils.service.ts @@ -331,7 +331,6 @@ export class DashboardUtilsService { const states = dashboardConfiguration.states; const state = states[targetState]; if (state) { - const allWidgets = dashboardConfiguration.widgets; const result: DashboardLayoutsInfo = {}; for (const l of Object.keys(state.layouts)) { const layout: DashboardLayout = state.layouts[l]; diff --git a/ui-ngx/src/app/core/services/item-buffer.service.ts b/ui-ngx/src/app/core/services/item-buffer.service.ts index 081953e2bf..2b0d95e3be 100644 --- a/ui-ngx/src/app/core/services/item-buffer.service.ts +++ b/ui-ngx/src/app/core/services/item-buffer.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { Dashboard, DashboardLayoutId } from '@app/shared/models/dashboard.models'; -import { EntityAlias, EntityAliasFilter, EntityAliases, EntityAliasInfo, AliasesInfo } from '@shared/models/alias.models'; +import { AliasesInfo, EntityAlias, EntityAliases, EntityAliasInfo } from '@shared/models/alias.models'; import { DatasourceType, Widget, WidgetPosition, WidgetSize } from '@shared/models/widget.models'; import { DashboardUtilsService } from '@core/services/dashboard-utils.service'; import { deepClone, isEqual } from '@core/utils'; diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 84799da5ed..6032bf24f3 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -23,7 +23,7 @@ import { take } from 'rxjs/operators'; import { HomeSection, MenuSection } from '@core/services/menu.models'; import { BehaviorSubject, Observable, Subject } from 'rxjs'; import { Authority } from '@shared/models/authority.enum'; -import {AuthUser} from '@shared/models/user.model'; +import { AuthUser } from '@shared/models/user.model'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/notification.service.ts b/ui-ngx/src/app/core/services/notification.service.ts index c4dd566d56..78a2cd198d 100644 --- a/ui-ngx/src/app/core/services/notification.service.ts +++ b/ui-ngx/src/app/core/services/notification.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { HideNotification, NotificationMessage } from '@app/core/notification/notification.models'; -import { BehaviorSubject, Observable, Subject } from 'rxjs'; +import { Observable, Subject } from 'rxjs'; @Injectable( diff --git a/ui-ngx/src/app/core/services/raf.service.ts b/ui-ngx/src/app/core/services/raf.service.ts index 98d31d7dd6..3461cbd9ee 100644 --- a/ui-ngx/src/app/core/services/raf.service.ts +++ b/ui-ngx/src/app/core/services/raf.service.ts @@ -15,7 +15,6 @@ /// import { Inject, Injectable, NgZone } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; import { WINDOW } from '@core/services/window.service'; export type CancelAnimationFrame = () => void; diff --git a/ui-ngx/src/app/core/services/script/node-script-test.service.ts b/ui-ngx/src/app/core/services/script/node-script-test.service.ts index ac50af85f8..d6272d58b7 100644 --- a/ui-ngx/src/app/core/services/script/node-script-test.service.ts +++ b/ui-ngx/src/app/core/services/script/node-script-test.service.ts @@ -15,11 +15,14 @@ /// import { Injectable } from '@angular/core'; -import { Observable, of } from 'rxjs'; +import { Observable } from 'rxjs'; import { RuleChainService } from '@core/http/rule-chain.service'; -import { map, switchMap } from 'rxjs/operators'; +import { switchMap } from 'rxjs/operators'; import { MatDialog } from '@angular/material/dialog'; -import { NodeScriptTestDialogComponent, NodeScriptTestDialogData } from '@shared/components/dialog/node-script-test-dialog.component'; +import { + NodeScriptTestDialogComponent, + NodeScriptTestDialogData +} from '@shared/components/dialog/node-script-test-dialog.component'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/time.service.ts b/ui-ngx/src/app/core/services/time.service.ts index ea9326e6ba..d76cfc9135 100644 --- a/ui-ngx/src/app/core/services/time.service.ts +++ b/ui-ngx/src/app/core/services/time.service.ts @@ -15,7 +15,14 @@ /// import { Injectable } from '@angular/core'; -import { AggregationType, DAY, defaultTimeIntervals, SECOND, Timewindow, defaultTimewindow } from '@shared/models/time/time.models'; +import { + AggregationType, + DAY, + defaultTimeIntervals, + defaultTimewindow, + SECOND, + Timewindow +} from '@shared/models/time/time.models'; import { HttpClient } from '@angular/common/http'; import { Observable } from 'rxjs'; import { defaultHttpOptions } from '@core/http/http-utils'; diff --git a/ui-ngx/src/app/core/services/window.service.ts b/ui-ngx/src/app/core/services/window.service.ts index 1bd0ce3c8e..acb11f7e9e 100644 --- a/ui-ngx/src/app/core/services/window.service.ts +++ b/ui-ngx/src/app/core/services/window.service.ts @@ -16,7 +16,7 @@ import { isPlatformBrowser } from '@angular/common'; -import { ClassProvider, FactoryProvider, InjectionToken, PLATFORM_ID, Injectable } from '@angular/core'; +import { ClassProvider, FactoryProvider, Injectable, InjectionToken, PLATFORM_ID } from '@angular/core'; /* Create a new injection token for injecting the window into a component. */ export const WINDOW = new InjectionToken('WindowToken'); diff --git a/ui-ngx/src/app/core/settings/settings.effects.ts b/ui-ngx/src/app/core/settings/settings.effects.ts index f11cc5bbed..e16e7b862b 100644 --- a/ui-ngx/src/app/core/settings/settings.effects.ts +++ b/ui-ngx/src/app/core/settings/settings.effects.ts @@ -14,27 +14,16 @@ /// limitations under the License. /// -import { ActivationEnd, ActivationStart, Router } from '@angular/router'; +import { ActivationEnd, Router } from '@angular/router'; import { Injectable } from '@angular/core'; import { select, Store } from '@ngrx/store'; import { Actions, Effect, ofType } from '@ngrx/effects'; import { TranslateService } from '@ngx-translate/core'; import { merge } from 'rxjs'; -import { - tap, - withLatestFrom, - map, - distinctUntilChanged, - filter -} from 'rxjs/operators'; +import { distinctUntilChanged, filter, map, tap, withLatestFrom } from 'rxjs/operators'; -import { - SettingsActionTypes, - SettingsActions, -} from './settings.actions'; -import { - selectSettingsState -} from './settings.selectors'; +import { SettingsActions, SettingsActionTypes, } from './settings.actions'; +import { selectSettingsState } from './settings.selectors'; import { AppState } from '@app/core/core.state'; import { LocalStorageService } from '@app/core/local-storage/local-storage.service'; import { TitleService } from '@app/core/services/title.service'; diff --git a/ui-ngx/src/app/core/translate/missing-translate-handler.ts b/ui-ngx/src/app/core/translate/missing-translate-handler.ts index dd91e9f4bf..32353fa2cf 100644 --- a/ui-ngx/src/app/core/translate/missing-translate-handler.ts +++ b/ui-ngx/src/app/core/translate/missing-translate-handler.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {MissingTranslationHandler, MissingTranslationHandlerParams} from '@ngx-translate/core'; +import { MissingTranslationHandler, MissingTranslationHandlerParams } from '@ngx-translate/core'; import { customTranslationsPrefix } from '@app/shared/models/constants'; export class TbMissingTranslationHandler implements MissingTranslationHandler { diff --git a/ui-ngx/src/app/core/translate/translate-default-compiler.ts b/ui-ngx/src/app/core/translate/translate-default-compiler.ts index 26fcfc0746..7badb08f12 100644 --- a/ui-ngx/src/app/core/translate/translate-default-compiler.ts +++ b/ui-ngx/src/app/core/translate/translate-default-compiler.ts @@ -15,10 +15,12 @@ /// import { - MESSAGE_FORMAT_CONFIG, MessageFormatConfig, + MESSAGE_FORMAT_CONFIG, + MessageFormatConfig, TranslateMessageFormatCompiler } from 'ngx-translate-messageformat-compiler'; import { Inject, Injectable, Optional } from '@angular/core'; + const parse = require('messageformat-parser').parse; @Injectable({ providedIn: 'root' }) diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index ea3682f72e..467f2e3e09 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -15,8 +15,8 @@ /// import _ from 'lodash'; -import { Observable, Subject, fromEvent, of } from 'rxjs'; -import { finalize, share, map } from 'rxjs/operators'; +import { fromEvent, Observable, of, Subject } from 'rxjs'; +import { finalize, map, share } from 'rxjs/operators'; import base64js from 'base64-js'; export function onParentScrollOrWindowResize(el: Node): Observable { diff --git a/ui-ngx/src/app/modules/dashboard/dashboard-pages.routing.module.ts b/ui-ngx/src/app/modules/dashboard/dashboard-pages.routing.module.ts index f7ce154661..d163f6a7b8 100644 --- a/ui-ngx/src/app/modules/dashboard/dashboard-pages.routing.module.ts +++ b/ui-ngx/src/app/modules/dashboard/dashboard-pages.routing.module.ts @@ -17,18 +17,12 @@ import { Injectable, NgModule } from '@angular/core'; import { ActivatedRouteSnapshot, Resolve, RouterModule, Routes } from '@angular/router'; -import {Authority} from '@shared/models/authority.enum'; +import { Authority } from '@shared/models/authority.enum'; import { DashboardPageComponent } from '@home/pages/dashboard/dashboard-page.component'; -import { BreadCrumbConfig, BreadCrumbLabelFunction } from '@shared/components/breadcrumb'; -import { widgetTypesBreadcumbLabelFunction } from '@home/pages/widget/widget-library-routing.module'; -import { WidgetsBundle } from '@shared/models/widgets-bundle.model'; -import { WidgetService } from '@core/http/widget.service'; -import { Observable } from 'rxjs'; import { Dashboard } from '@app/shared/models/dashboard.models'; import { DashboardService } from '@core/http/dashboard.service'; import { DashboardUtilsService } from '@core/services/dashboard-utils.service'; -import { map } from 'rxjs/operators'; -import { dashboardBreadcumbLabelFunction, DashboardResolver } from '@app/modules/home/pages/dashboard/dashboard-routing.module'; +import { DashboardResolver } from '@app/modules/home/pages/dashboard/dashboard-routing.module'; import { UtilsService } from '@core/services/utils.service'; import { Widget } from '@app/shared/models/widget.models'; diff --git a/ui-ngx/src/app/modules/dashboard/dashboard-routing.module.ts b/ui-ngx/src/app/modules/dashboard/dashboard-routing.module.ts index 5166465170..6f0c208596 100644 --- a/ui-ngx/src/app/modules/dashboard/dashboard-routing.module.ts +++ b/ui-ngx/src/app/modules/dashboard/dashboard-routing.module.ts @@ -15,7 +15,7 @@ /// import { NgModule } from '@angular/core'; -import { Routes, RouterModule } from '@angular/router'; +import { RouterModule, Routes } from '@angular/router'; import { AuthGuard } from '@core/guards/auth.guard'; import { StoreModule } from '@ngrx/store'; diff --git a/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html b/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html index 2cdc3faf28..ad85a91923 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html @@ -57,7 +57,7 @@
+ style="margin-bottom: 0; padding-right: 20px;"> {{ 'dashboard.open-dashboard' | translate }}
-
+
diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-table-header.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-table-header.component.ts index e24345bc7e..ba04af086f 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-table-header.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-table-header.component.ts @@ -16,13 +16,14 @@ import { BaseData, HasId } from '@shared/models/base-data'; import { PageComponent } from '@shared/components/page.component'; -import { Input, OnInit, Directive } from '@angular/core'; +import { Directive, Input, OnInit } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; import { PageLink } from '@shared/models/page/page-link'; @Directive() +// tslint:disable-next-line:directive-class-suffix export abstract class EntityTableHeaderComponent, P extends PageLink = PageLink, L extends BaseData = T, diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts index 31ed87529e..41ba7420dc 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts @@ -16,15 +16,14 @@ import { BaseData, HasId } from '@shared/models/base-data'; import { PageComponent } from '@shared/components/page.component'; -import { AfterViewInit, ContentChildren, EventEmitter, Input, OnInit, Output, QueryList, ViewChildren, Directive } from '@angular/core'; +import { AfterViewInit, Directive, Input, OnInit, QueryList, ViewChildren } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; import { MatTab } from '@angular/material/tabs'; -import { EntityAction } from '@home/models/entity/entity-component.models'; import { BehaviorSubject } from 'rxjs'; import { Authority } from '@app/shared/models/authority.enum'; -import { selectAuthUser, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { getCurrentAuthUser } from '@core/auth/auth.selectors'; import { AuthUser } from '@shared/models/user.model'; import { EntityType } from '@shared/models/entity-type.models'; import { AuditLogMode } from '@shared/models/audit-log.models'; @@ -35,6 +34,7 @@ import { NgForm } from '@angular/forms'; import { PageLink } from '@shared/models/page/page-link'; @Directive() +// tslint:disable-next-line:directive-class-suffix export abstract class EntityTabsComponent, P extends PageLink = PageLink, L extends BaseData = T, diff --git a/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.ts b/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.ts index 20f7a1ee9b..ba0e66fc2e 100644 --- a/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.ts @@ -19,27 +19,10 @@ import { ErrorStateMatcher } from '@angular/material/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { - AbstractControl, - FormArray, - FormBuilder, - FormControl, - FormGroup, - FormGroupDirective, - NgForm, Validators, ValidatorFn -} from '@angular/forms'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@app/shared/components/dialog.component'; -import { AttributeData } from '@shared/models/telemetry/telemetry.models'; -import { EntityAlias, EntityAliases, EntityAliasFilter } from '@shared/models/alias.models'; -import { DatasourceType, Widget, widgetType } from '@shared/models/widget.models'; -import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; -import { UtilsService } from '@core/services/utils.service'; -import { TranslateService } from '@ngx-translate/core'; import { ActionNotificationShow } from '@core/notification/notification.actions'; -import { DialogService } from '@core/services/dialog.service'; -import { EntityService } from '@core/http/entity.service'; -import { Observable, of } from 'rxjs'; export interface ImportDialogData { importTitle: string; diff --git a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.scss b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.scss index 6a37c45ae6..32d865eaa4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.scss @@ -17,7 +17,7 @@ mat-form-field.resource-field { max-height: 40px; - margin: 10px 0px 0px 0px; + margin: 10px 0 0; .mat-form-field-wrapper { padding-bottom: 0; .mat-form-field-flex { diff --git a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.scss b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.scss index afd505b54e..1a58e8c6f8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.scss @@ -51,7 +51,7 @@ width: 24px; height: 24px; font-size: 24px; - margin-left: 0px; + margin-left: 0; color: inherit; opacity: inherit; } diff --git a/ui-ngx/src/app/modules/home/components/widget/dialog/custom-dialog.service.ts b/ui-ngx/src/app/modules/home/components/widget/dialog/custom-dialog.service.ts index 98f07663f3..b9e4513279 100644 --- a/ui-ngx/src/app/modules/home/components/widget/dialog/custom-dialog.service.ts +++ b/ui-ngx/src/app/modules/home/components/widget/dialog/custom-dialog.service.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Injectable, Injector, NgModule } from '@angular/core'; +import { Injectable, NgModule } from '@angular/core'; import { Observable } from 'rxjs'; import { MatDialog } from '@angular/material/dialog'; import { TranslateService } from '@ngx-translate/core'; @@ -36,7 +36,6 @@ export class CustomDialogService { private translate: TranslateService, private authService: AuthService, private dynamicComponentFactoryService: DynamicComponentFactoryService, - private injector: Injector, public dialog: MatDialog ) { } diff --git a/ui-ngx/src/app/modules/home/components/widget/legend-config-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/legend-config-panel.component.ts index a96d1bab9c..5f5abff7a3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend-config-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/legend-config-panel.component.ts @@ -15,7 +15,7 @@ /// import { Component, Inject, InjectionToken, OnInit, ViewContainerRef } from '@angular/core'; -import { Overlay, OverlayRef } from '@angular/cdk/overlay'; +import { OverlayRef } from '@angular/cdk/overlay'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -60,7 +60,6 @@ export class LegendConfigPanelComponent extends PageComponent implements OnInit public overlayRef: OverlayRef, protected store: Store, public fb: FormBuilder, - private overlay: Overlay, public viewContainerRef: ViewContainerRef) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 6f84149ae3..ee8f72048b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -37,8 +37,7 @@ import { deepClone, isDefined, isNumber } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; -import { DataSource } from '@angular/cdk/collections'; -import { CollectionViewer, SelectionModel } from '@angular/cdk/collections'; +import { CollectionViewer, DataSource, SelectionModel } from '@angular/cdk/collections'; import { BehaviorSubject, forkJoin, fromEvent, merge, Observable, of } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { entityTypeTranslations } from '@shared/models/entity-type.models'; @@ -59,7 +58,8 @@ import { getColumnWidth, TableWidgetDataKeySettings, TableWidgetSettings, - toAlarmColumnDef, widthStyle + toAlarmColumnDef, + widthStyle } from '@home/components/widget/lib/table-widget.models'; import { ConnectedPosition, Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ComponentPortal, PortalInjector } from '@angular/cdk/portal'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/analogue-radial-gauge.ts b/ui-ngx/src/app/modules/home/components/widget/lib/analogue-radial-gauge.ts index 38ddc7f536..1571c5c3d0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/analogue-radial-gauge.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/analogue-radial-gauge.ts @@ -16,7 +16,8 @@ import * as CanvasGauges from 'canvas-gauges'; import { - AnalogueRadialGaugeSettings, getAnalogueRadialGaugeSettingsSchema + AnalogueRadialGaugeSettings, + getAnalogueRadialGaugeSettingsSchema } from '@home/components/widget/lib/analogue-radial-gauge.models'; import { JsonSettingsSchema } from '@shared/models/widget.models'; import { WidgetContext } from '@home/models/widget-component.models'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.html index da922d8ca4..94d66e2a8a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.html @@ -19,7 +19,7 @@ [ngClass]="{'short-mode':ctx.width < 400, 'labels-hidden': settings.hideLabels, 'long-mode': ctx.width >= 400}"> widgets.date-range-navigator.localizationMap.Date picker - + {{advancedModel.chosenLabel}} @@ -27,7 +27,7 @@ widgets.date-range-navigator.localizationMap.Interval - + {{'widgets.date-range-navigator.localizationMap.' + customInterval.label | translate}} @@ -43,7 +43,7 @@ widgets.date-range-navigator.localizationMap.Step size - + {{'widgets.date-range-navigator.localizationMap.' + date.value.label | translate}} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.scss b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.scss index 5663801189..e3505275e5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.scss @@ -34,12 +34,6 @@ margin: 4px 0; } - .navigation { - md-input-container { - margin: 0; - } - } - .picker { .picker__wrapper { position: relative; @@ -74,10 +68,6 @@ .drn__element { width: 100%; - - md-input-container { - flex: 1; - } } .picker { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.ts index a0893305e3..8d0cc7cb3a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator.component.ts @@ -38,7 +38,7 @@ import { DateRangeNavigatorSettings, getFormattedDate } from '@home/components/widget/lib/date-range-navigator/date-range-navigator.models'; -import { DatePipe, KeyValue } from '@angular/common'; +import { KeyValue } from '@angular/common'; import * as _moment from 'moment'; import { ConnectedPosition, Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ComponentPortal, PortalInjector } from '@angular/cdk/portal'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index fa2fe59903..bbdb145835 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -43,9 +43,8 @@ import { deepClone, isDefined, isNumber } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; -import { DataSource } from '@angular/cdk/collections'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; -import { CollectionViewer } from '@angular/cdk/collections'; import { BehaviorSubject, fromEvent, merge, Observable, of } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -55,20 +54,21 @@ import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { - CellContentInfo, - CellStyleInfo, - constructTableCssString, - DisplayColumn, - EntityColumn, - EntityData, - fromEntityColumnDef, - getCellContentInfo, - getCellStyleInfo, - getColumnWidth, - getEntityValue, - TableWidgetDataKeySettings, - TableWidgetSettings, - toEntityColumnDef, widthStyle + CellContentInfo, + CellStyleInfo, + constructTableCssString, + DisplayColumn, + EntityColumn, + EntityData, + fromEntityColumnDef, + getCellContentInfo, + getCellStyleInfo, + getColumnWidth, + getEntityValue, + TableWidgetDataKeySettings, + TableWidgetSettings, + toEntityColumnDef, + widthStyle } from '@home/components/widget/lib/table-widget.models'; import { ConnectedPosition, Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ComponentPortal, PortalInjector } from '@angular/cdk/portal'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts index 765d47e8f4..6726931576 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts @@ -17,7 +17,7 @@ // tslint:disable-next-line:no-reference /// -import { JsonSettingsSchema, DataKey, DatasourceData, Datasource } from '@shared/models/widget.models'; +import { DataKey, Datasource, DatasourceData, JsonSettingsSchema } from '@shared/models/widget.models'; import * as moment_ from 'moment'; export declare type ChartType = 'line' | 'pie' | 'bar' | 'state' | 'graph'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts index da870813f3..956e37b51b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts @@ -14,16 +14,23 @@ /// limitations under the License. /// -import L, { LatLngTuple, LatLngBounds, Point } from 'leaflet'; +import L, { LatLngBounds, LatLngTuple } from 'leaflet'; import 'leaflet-providers'; import 'leaflet.markercluster/dist/MarkerCluster.css'; import 'leaflet.markercluster/dist/MarkerCluster.Default.css'; import LM from 'leaflet.markercluster/dist/leaflet.markercluster'; -import { MapSettings, MarkerSettings, FormattedData, UnitedMapSettings, PolygonSettings, PolylineSettings } from './map-models'; +import { + FormattedData, + MapSettings, + MarkerSettings, + PolygonSettings, + PolylineSettings, + UnitedMapSettings +} from './map-models'; import { Marker } from './markers'; -import { Observable, BehaviorSubject } from 'rxjs'; +import { BehaviorSubject, Observable } from 'rxjs'; import { filter } from 'rxjs/operators'; import { Polyline } from './polyline'; import { Polygon } from './polygon'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts index 69faafcb14..0b7a92455d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts @@ -17,22 +17,22 @@ import { MapProviders, UnitedMapSettings } from './map-models'; import LeafletMap from './leaflet-map'; import { - openstreetMapSettingsSchema, - googleMapSettingsSchema, - imageMapSettingsSchema, - tencentMapSettingsSchema, - commonMapSettingsSchema, - routeMapSettingsSchema, - markerClusteringSettingsSchema, - markerClusteringSettingsSchemaLeaflet, - hereMapSettingsSchema, - mapProviderSchema, - mapPolygonSchema + commonMapSettingsSchema, + googleMapSettingsSchema, + hereMapSettingsSchema, + imageMapSettingsSchema, + mapPolygonSchema, + mapProviderSchema, + markerClusteringSettingsSchema, + markerClusteringSettingsSchemaLeaflet, + openstreetMapSettingsSchema, + routeMapSettingsSchema, + tencentMapSettingsSchema } from './schemes'; -import { MapWidgetStaticInterface, MapWidgetInterface } from './map-widget.interface'; -import { OpenStreetMap, TencentMap, GoogleMap, HEREMap, ImageMap } from './providers'; -import { parseFunction, parseArray, parseData, safeExecute, parseWithTranslation } from '@core/utils'; -import { initSchema, addToSchema, mergeSchemes, addCondition, addGroupInfo } from '@core/schema-utils'; +import { MapWidgetInterface, MapWidgetStaticInterface } from './map-widget.interface'; +import { GoogleMap, HEREMap, ImageMap, OpenStreetMap, TencentMap } from './providers'; +import { parseArray, parseData, parseFunction, parseWithTranslation } from '@core/utils'; +import { addCondition, addGroupInfo, addToSchema, initSchema, mergeSchemes } from '@core/schema-utils'; import { forkJoin } from 'rxjs'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; import { getDefCenterPosition } from './maps-utils'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts index 4ac437830c..f3c825c069 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts @@ -15,9 +15,7 @@ /// import L from 'leaflet'; -import _ from 'lodash'; -import { MarkerSettings, PolylineSettings, PolygonSettings } from './map-models'; -import { parseWithTranslation } from '@app/core/utils'; +import { MarkerSettings, PolygonSettings, PolylineSettings } from './map-models'; export function createTooltip(target: L.Layer, settings: MarkerSettings | PolylineSettings | PolygonSettings, @@ -63,4 +61,4 @@ export function getDefCenterPosition(position) { if (typeof (position) === 'object') return position; return [0, 0]; -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss index d402a4b5d9..8cb1894194 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.scss @@ -16,10 +16,8 @@ .arrow { height: 30px; width: 30px; - background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAtCAMAAAANxBKoAAACAVBMVEUAAAAAqv9AgP8ggP8udP8qgP8ndv8kbf8kgP8eeP8rcf8mc/8jdP8hev8me/8kdv8jcv8pcP8ic/8neP8leP8ofP8mef8pev8nef8oe/8nef8pef8qfv8qgf8qe/8mev8off8oev8oef8oe/8pev8qe/8oe/8off8pfP8oev8pev8qe/8qfP8pev8pe/8qfP8pe/8oev8pe/8qe/8qfP8qfP8qff8qff8qfP8qff8pe/8qff8qfP8qfP8pev8pev8JYf8KYP8UY/8UZf8WY/8WZv8YZP8bZP8fZv8fZ/8gZv8gZ/8hZP8iZ/8jaf8jav8kaP8kav8ka/8kbP8kbf8kb/8lbP8lbf8lbv8lb/8lcP8mb/8mcP8mcf8ncf8ncv8nc/8ndP8ndf8ocv4odP8odf8odv8od/8oeP8peP8pef8pev8qev8qe/8qfP8rbP8rfP8rff8tc/4vcP4xbv4zcf42dP49df4+gP4/fv5Rif5biP1ul/14mvx4nvyMrfuMrvuWsfuctvuguPuww/vD0fnH1vrI1frK2PrL2fnV3vnb4vnc5fnd5fnd5vne5fnj6vnl6/jm7Pno7fXo7fbo7ffo7fjp7fnp7vjq7vjs8fnt8Pjt8fjw8fXw8fbw8fjy8/jz9fn5+fj7+/f++/j/+/X/+/b/+/f/+/j//Pj//vhoUaHMAAAAQHRSTlMAAwQICwwNDg4REhQWFxscHTI1QkRGSktOU1RdYWFscXJzhYWKqre6wMHDw8PHx8rN09TW1uLi7e/x8vLz9fj5hfwW0QAAAnVJREFUSMeV1fdb00AYB/BzoQjurSjuvQcuFLcgWqTVtLVpL0GS2GuDuLeouPeAiLOuwOFf6WU0ucvq4/d52v6Qz/M+18u97wHgyfh5m5vUYlHt2rV8BojOqGUqonNhVgTegfxZE2JXoMAUFgXYsSoKy3YfnoIiUqxm8UwUnSoaT0WVUuPi6lCUz+ft/+pqNYwqsiwrlm8s45XBVpE7BQgFyeZLbB1ihVwmefjpDSjbi7HwziArCdlU/MArjDHfaestBh4dZs9/JBb/4kVbo5FEr/JaWRL4VPvZ39jMVV4q63qiCx7bIfDJtst/LYt/JnLldaNuACbS9JwiQj4Zu/3HlDr5nEyJivMcgPn0Nogwfaq1Z9igQ7pO9AuqNEJzQINTV5ZghjvywLFG6dYMVRptAF3uPvBc8xPsWH0Q44ccpEqjvaBoa0lI7/ts2kHdCsbDJ5jSSC2/SEU8856x+hDGd+JZujQquvp6ieT7Nyel0o/jCSgzTefq0y81I30f7PRr2i1fbWrd+98yXtMGfOt29wTyiebHpu83dZ+m3U969qSBeje5DHeod8Dx5PeYZ7/rPO8y0XLvk+1J8UcJyLzLScw5kTsgz8Xu2vXJV0uaOSfsGcznZZGcwdi1L5qV13HmDILV3tY1z3fbxa8Wv+Se74VEj/H3jmz0TvuVN4Z+x/ZORF8efU64s/CtUT0vCbk0d/DZTSgxPQ/WhswTSYTZrDNPllaaVQo1q/Y4k63mv+YgmFZxxk6gR/LsCngcO+6nR9lCrfcqqQ2/dxqD7rT1IXhxyB24O8BuCr9eq9axU7F7wYjoy3ty3TZVLaCC2rRxru/hPwVgtEdFK5vOAAAAAElFTkSuQmCC"); background-size: cover; - background-repeat: no-repeat; - background-position: center center; + background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAtCAMAAAANxBKoAAACAVBMVEUAAAAAqv9AgP8ggP8udP8qgP8ndv8kbf8kgP8eeP8rcf8mc/8jdP8hev8me/8kdv8jcv8pcP8ic/8neP8leP8ofP8mef8pev8nef8oe/8nef8pef8qfv8qgf8qe/8mev8off8oev8oef8oe/8pev8qe/8oe/8off8pfP8oev8pev8qe/8qfP8pev8pe/8qfP8pe/8oev8pe/8qe/8qfP8qfP8qff8qff8qfP8qff8pe/8qff8qfP8qfP8pev8pev8JYf8KYP8UY/8UZf8WY/8WZv8YZP8bZP8fZv8fZ/8gZv8gZ/8hZP8iZ/8jaf8jav8kaP8kav8ka/8kbP8kbf8kb/8lbP8lbf8lbv8lb/8lcP8mb/8mcP8mcf8ncf8ncv8nc/8ndP8ndf8ocv4odP8odf8odv8od/8oeP8peP8pef8pev8qev8qe/8qfP8rbP8rfP8rff8tc/4vcP4xbv4zcf42dP49df4+gP4/fv5Rif5biP1ul/14mvx4nvyMrfuMrvuWsfuctvuguPuww/vD0fnH1vrI1frK2PrL2fnV3vnb4vnc5fnd5fnd5vne5fnj6vnl6/jm7Pno7fXo7fbo7ffo7fjp7fnp7vjq7vjs8fnt8Pjt8fjw8fXw8fbw8fjy8/jz9fn5+fj7+/f++/j/+/X/+/b/+/f/+/j//Pj//vhoUaHMAAAAQHRSTlMAAwQICwwNDg4REhQWFxscHTI1QkRGSktOU1RdYWFscXJzhYWKqre6wMHDw8PHx8rN09TW1uLi7e/x8vLz9fj5hfwW0QAAAnVJREFUSMeV1fdb00AYB/BzoQjurSjuvQcuFLcgWqTVtLVpL0GS2GuDuLeouPeAiLOuwOFf6WU0ucvq4/d52v6Qz/M+18u97wHgyfh5m5vUYlHt2rV8BojOqGUqonNhVgTegfxZE2JXoMAUFgXYsSoKy3YfnoIiUqxm8UwUnSoaT0WVUuPi6lCUz+ft/+pqNYwqsiwrlm8s45XBVpE7BQgFyeZLbB1ihVwmefjpDSjbi7HwziArCdlU/MArjDHfaestBh4dZs9/JBb/4kVbo5FEr/JaWRL4VPvZ39jMVV4q63qiCx7bIfDJtst/LYt/JnLldaNuACbS9JwiQj4Zu/3HlDr5nEyJivMcgPn0Nogwfaq1Z9igQ7pO9AuqNEJzQINTV5ZghjvywLFG6dYMVRptAF3uPvBc8xPsWH0Q44ccpEqjvaBoa0lI7/ts2kHdCsbDJ5jSSC2/SEU8856x+hDGd+JZujQquvp6ieT7Nyel0o/jCSgzTefq0y81I30f7PRr2i1fbWrd+98yXtMGfOt29wTyiebHpu83dZ+m3U969qSBeje5DHeod8Dx5PeYZ7/rPO8y0XLvk+1J8UcJyLzLScw5kTsgz8Xu2vXJV0uaOSfsGcznZZGcwdi1L5qV13HmDILV3tY1z3fbxa8Wv+Se74VEj/H3jmz0TvuVN4Z+x/ZORF8efU64s/CtUT0vCbk0d/DZTSgxPQ/WhswTSYTZrDNPllaaVQo1q/Y4k63mv+YgmFZxxk6gR/LsCngcO+6nR9lCrfcqqQ2/dxqD7rT1IXhxyB24O8BuCr9eq9axU7F7wYjoy3ty3TZVLaCC2rRxru/hPwVgtEdFK5vOAAAAAElFTkSuQmCC") no-repeat center center; } .leaflet-div-icon, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts index 0b0d269ce1..f5263bb4a3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts @@ -17,7 +17,7 @@ import L from 'leaflet'; import LeafletMap from '../leaflet-map'; -import { MapSettings, UnitedMapSettings } from '../map-models'; +import { UnitedMapSettings } from '../map-models'; import 'leaflet.gridlayer.googlemutant'; let googleLoaded = false; @@ -54,4 +54,4 @@ export class GoogleMap extends LeafletMap { document.getElementsByTagName('head')[0].appendChild(script); } } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts index 2b48e3ee8b..9ad0d7b160 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/here-map.ts @@ -16,7 +16,7 @@ import L from 'leaflet'; import LeafletMap from '../leaflet-map'; -import { MapSettings, UnitedMapSettings } from '../map-models'; +import { UnitedMapSettings } from '../map-models'; export class HEREMap extends LeafletMap { constructor($container, options: UnitedMapSettings) { @@ -27,4 +27,4 @@ export class HEREMap extends LeafletMap { super.setMap(map); super.initSettings(options); } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts index 94c58b6520..8a7c8f5712 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts @@ -16,7 +16,7 @@ import L, { LatLngLiteral } from 'leaflet'; import LeafletMap from '../leaflet-map'; -import { MapSettings, UnitedMapSettings } from '../map-models'; +import { UnitedMapSettings } from '../map-models'; import { aspectCache } from '@app/core/utils'; const maxZoom = 4;// ? @@ -135,4 +135,4 @@ export class ImageMap extends LeafletMap { [this.options.yPosKeyName]: (position.lat + 180) / 360 } }*/ -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/openstreet-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/openstreet-map.ts index e39aa6ab02..38db40208a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/openstreet-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/openstreet-map.ts @@ -16,7 +16,7 @@ import L from 'leaflet'; import LeafletMap from '../leaflet-map'; -import { MapSettings, UnitedMapSettings } from '../map-models'; +import { UnitedMapSettings } from '../map-models'; export class OpenStreetMap extends LeafletMap { constructor($container, options: UnitedMapSettings) { @@ -27,4 +27,4 @@ export class OpenStreetMap extends LeafletMap { super.setMap(map); super.initSettings(options); } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/tencent-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/tencent-map.ts index 980846796c..51c848c533 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/tencent-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/tencent-map.ts @@ -17,7 +17,7 @@ import L from 'leaflet'; import LeafletMap from '../leaflet-map'; -import { MapSettings, UnitedMapSettings } from '../map-models'; +import { UnitedMapSettings } from '../map-models'; export class TencentMap extends LeafletMap { constructor($container, options: UnitedMapSettings) { @@ -29,4 +29,4 @@ export class TencentMap extends LeafletMap { super.setMap(map); super.initSettings(options); } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts index e15d25bd86..0b9a8cb995 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts @@ -23,8 +23,8 @@ import { AppState } from '@core/core.state'; import { isDefined, isNumber } from '@core/utils'; import { CanvasDigitalGauge, CanvasDigitalGaugeOptions } from '@home/components/widget/lib/canvas-digital-gauge'; import * as tinycolor_ from 'tinycolor2'; -import GenericOptions = CanvasGauges.GenericOptions; import { ResizeObserver } from '@juggle/resize-observer'; +import GenericOptions = CanvasGauges.GenericOptions; const tinycolor = tinycolor_; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.scss b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.scss index 2f393f5856..d20588e412 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.scss @@ -32,16 +32,16 @@ .mat-row:not(.mat-row-select), .mat-header-row:not(.mat-row-select) { mat-cell:nth-child(n+2):nth-last-child(n+2), mat-footer-cell:nth-child(n+2):nth-last-child(n+2), mat-header-cell:nth-child(n+2):nth-last-child(n+2) { - padding: 0px 5px; + padding: 0 5px; } } .mat-row.mat-row-select, .mat-header-row.mat-row-select { mat-cell:nth-child(2), mat-footer-cell:nth-child(2), mat-header-cell:nth-child(2) { - padding: 0px 5px; + padding: 0 5px; } mat-cell:nth-child(n+3):nth-last-child(n+2), mat-footer-cell:nth-child(n+3):nth-last-child(n+2), mat-header-cell:nth-child(n+3):nth-last-child(n+2) { - padding: 0px 5px; + padding: 0 5px; } } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index d0bb73e3af..592cb4d286 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -44,8 +44,7 @@ import { isDefined, isNumber } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; -import { DataSource } from '@angular/cdk/collections'; -import { CollectionViewer } from '@angular/cdk/collections'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { BehaviorSubject, fromEvent, merge, Observable, of } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; diff --git a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts index 5d2f8bfa2e..ce1502ac58 100644 --- a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts @@ -19,15 +19,15 @@ import _ from 'lodash'; import tinycolor from 'tinycolor2'; import { interpolateOnPointSegment } from 'leaflet-geometryutil'; -import { Component, OnInit, Input, ViewChild, AfterViewInit, ChangeDetectorRef, SecurityContext } from '@angular/core'; +import { AfterViewInit, ChangeDetectorRef, Component, Input, OnInit, SecurityContext, ViewChild } from '@angular/core'; import { MapWidgetController, TbMapWidgetV2 } from '../lib/maps/map-widget2'; import { MapProviders } from '../lib/maps/map-models'; -import { parseArray, parseWithTranslation, safeExecute, parseTemplate } from '@app/core/utils'; -import { initSchema, addToSchema, addGroupInfo } from '@app/core/schema-utils'; +import { parseArray, parseWithTranslation, safeExecute } from '@app/core/utils'; +import { addGroupInfo, addToSchema, initSchema } from '@app/core/schema-utils'; import { tripAnimationSchema } from '../lib/maps/schemes'; import { DomSanitizer } from '@angular/platform-browser'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; -import { getRatio, findAngle } from '../lib/maps/maps-utils'; +import { findAngle, getRatio } from '../lib/maps/maps-utils'; import { JsonSettingsSchema, WidgetConfig } from '@shared/models/widget.models'; import moment from 'moment'; diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts index 344f7cf02d..00ba5b2b8d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts @@ -23,7 +23,9 @@ import { Datasource, DatasourceType, datasourceTypeTranslationMap, - defaultLegendConfig, GroupInfo, JsonSchema, + defaultLegendConfig, + GroupInfo, + JsonSchema, widgetType } from '@shared/models/widget.models'; import { diff --git a/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.ts b/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.ts index dd696d25bd..d7e03bc79f 100644 --- a/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.ts @@ -120,16 +120,12 @@ export class AddEntitiesToCustomerDialogComponent extends switch (this.data.entityType) { case EntityType.DEVICE: return this.deviceService.assignDeviceToCustomer(customerId, entityId); - break; case EntityType.ASSET: return this.assetService.assignAssetToCustomer(customerId, entityId); - break; case EntityType.ENTITY_VIEW: return this.entityViewService.assignEntityViewToCustomer(customerId, entityId); - break; case EntityType.DASHBOARD: return this.dashboardService.assignDashboardToCustomer(customerId, entityId); - break; } } diff --git a/ui-ngx/src/app/modules/home/dialogs/assign-to-customer-dialog.component.ts b/ui-ngx/src/app/modules/home/dialogs/assign-to-customer-dialog.component.ts index 1643223ebf..97177378f6 100644 --- a/ui-ngx/src/app/modules/home/dialogs/assign-to-customer-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/dialogs/assign-to-customer-dialog.component.ts @@ -114,13 +114,10 @@ export class AssignToCustomerDialogComponent extends switch (this.data.entityType) { case EntityType.DEVICE: return this.deviceService.assignDeviceToCustomer(customerId, entityId); - break; case EntityType.ASSET: return this.assetService.assignAssetToCustomer(customerId, entityId); - break; case EntityType.ENTITY_VIEW: return this.entityViewService.assignEntityViewToCustomer(customerId, entityId); - break; } } diff --git a/ui-ngx/src/app/modules/home/home-routing.module.ts b/ui-ngx/src/app/modules/home/home-routing.module.ts index 7aa99e6ebc..f73528db68 100644 --- a/ui-ngx/src/app/modules/home/home-routing.module.ts +++ b/ui-ngx/src/app/modules/home/home-routing.module.ts @@ -15,7 +15,7 @@ /// import { NgModule } from '@angular/core'; -import { Routes, RouterModule } from '@angular/router'; +import { RouterModule, Routes } from '@angular/router'; import { HomeComponent } from './home.component'; import { AuthGuard } from '@core/guards/auth.guard'; diff --git a/ui-ngx/src/app/modules/home/menu/menu-link.component.ts b/ui-ngx/src/app/modules/home/menu/menu-link.component.ts index 787d7b7928..6edef6a8af 100644 --- a/ui-ngx/src/app/modules/home/menu/menu-link.component.ts +++ b/ui-ngx/src/app/modules/home/menu/menu-link.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit, ViewEncapsulation } from '@angular/core'; +import { Component, Input, OnInit } from '@angular/core'; import { MenuSection } from '@core/services/menu.models'; @Component({ diff --git a/ui-ngx/src/app/modules/home/menu/menu-toggle.component.ts b/ui-ngx/src/app/modules/home/menu/menu-toggle.component.ts index fcadd04a2f..4054bc4b4f 100644 --- a/ui-ngx/src/app/modules/home/menu/menu-toggle.component.ts +++ b/ui-ngx/src/app/modules/home/menu/menu-toggle.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit, ViewEncapsulation } from '@angular/core'; +import { Component, Input, OnInit } from '@angular/core'; import { MenuSection } from '@core/services/menu.models'; import { Router } from '@angular/router'; diff --git a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts index 768b7fcc52..b79c16527e 100644 --- a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts @@ -15,13 +15,13 @@ /// import { GridsterComponent, GridsterConfig, GridsterItem, GridsterItemComponentInterface } from 'angular-gridster2'; -import { Widget, widgetType, WidgetPosition } from '@app/shared/models/widget.models'; +import { Widget, WidgetPosition, widgetType } from '@app/shared/models/widget.models'; import { WidgetLayout, WidgetLayouts } from '@app/shared/models/dashboard.models'; import { IDashboardWidget, WidgetAction, WidgetContext, WidgetHeaderAction } from './widget-component.models'; import { Timewindow } from '@shared/models/time/time.models'; import { Observable, of, Subject } from 'rxjs'; import { guid, isDefined, isEqual, isUndefined } from '@app/core/utils'; -import { IterableDiffer, KeyValueDiffer, NgZone } from '@angular/core'; +import { IterableDiffer, KeyValueDiffer } from '@angular/core'; import { IAliasController, IStateController } from '@app/core/api/widget-api.models'; export interface WidgetsData { diff --git a/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts b/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts index d377fe9373..3d46a8a8cf 100644 --- a/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts +++ b/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts @@ -14,10 +14,9 @@ /// limitations under the License. /// -import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { CollectionViewer, DataSource, SelectionModel } from '@angular/cdk/collections'; import { BehaviorSubject, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; -import { SelectionModel } from '@angular/cdk/collections'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, map, publishReplay, refCount, take, tap } from 'rxjs/operators'; import { EntityId } from '@app/shared/models/id/entity-id'; @@ -26,8 +25,8 @@ import { AttributeData, AttributeScope, isClientSideTelemetryType, - TelemetryType, - TelemetrySubscriber + TelemetrySubscriber, + TelemetryType } from '@shared/models/telemetry/telemetry.models'; import { AttributeService } from '@core/http/attribute.service'; import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service'; diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-component.models.ts index dde03b6227..9b081431ef 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-component.models.ts @@ -14,8 +14,8 @@ /// limitations under the License. /// -import {BaseData, HasId} from '@shared/models/base-data'; -import {EntityTableConfig} from './entities-table-config.models'; +import { BaseData, HasId } from '@shared/models/base-data'; +import { EntityTableConfig } from './entities-table-config.models'; export interface AddEntityDialogData> { entitiesTableConfig: EntityTableConfig; diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 185074a8ad..30dea45d6b 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -17,7 +17,10 @@ import { IDashboardComponent } from '@home/models/dashboard-component.models'; import { DataSet, - Datasource, DatasourceData, + Datasource, + DatasourceData, + JsonSettingsSchema, + Widget, WidgetActionDescriptor, WidgetActionSource, WidgetConfig, @@ -25,8 +28,7 @@ import { WidgetType, widgetType, WidgetTypeDescriptor, - WidgetTypeParameters, - Widget, JsonSettingsSchema + WidgetTypeParameters } from '@shared/models/widget.models'; import { Timewindow, WidgetTimewindow } from '@shared/models/time/time.models'; import { @@ -34,10 +36,11 @@ import { IStateController, IWidgetSubscription, IWidgetUtils, - RpcApi, SubscriptionEntityInfo, SubscriptionInfo, + RpcApi, + SubscriptionEntityInfo, TimewindowFunctions, WidgetActionsApi, - WidgetSubscriptionApi, WidgetSubscriptionContext, WidgetSubscriptionOptions + WidgetSubscriptionApi } from '@core/api/widget-api.models'; import { ChangeDetectorRef, ComponentFactory, Injector, NgZone, Type } from '@angular/core'; import { HttpErrorResponse } from '@angular/common/http'; @@ -45,13 +48,8 @@ import { RafService } from '@core/services/raf.service'; import { WidgetTypeId } from '@shared/models/id/widget-type-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetLayout } from '@shared/models/dashboard.models'; -import { DeviceService } from '@core/http/device.service'; -import { AssetService } from '@app/core/http/asset.service'; -import { DialogService } from '@core/services/dialog.service'; -import { CustomDialogService } from '@home/components/widget/dialog/custom-dialog.service'; -import { isDefined, formatValue } from '@core/utils'; -import { forkJoin, Observable, of, ReplaySubject } from 'rxjs'; -import { WidgetSubscription } from '@core/api/widget-subscription'; +import { formatValue, isDefined } from '@core/utils'; +import { forkJoin, of } from 'rxjs'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.ts index f69a9b591f..ab5535a92f 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.ts @@ -26,8 +26,10 @@ import { DashboardConfiguration, DashboardLayoutId, DashboardLayoutInfo, - DashboardLayoutsInfo, DashboardState, - DashboardStateLayouts, GridSettings, + DashboardLayoutsInfo, + DashboardState, + DashboardStateLayouts, + GridSettings, WidgetLayout } from '@app/shared/models/dashboard.models'; import { WINDOW } from '@core/services/window.service'; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.models.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.models.ts index 7fd00cdd04..010c64b444 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.models.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.models.ts @@ -70,7 +70,6 @@ export declare type DashboardPageLayouts = {[key in DashboardLayoutId]: Dashboar export class LayoutWidgetsArray implements Iterable { private widgetIds: string[] = []; - private pointer = 0; private loaded = false; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-widget-select.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-widget-select.component.ts index 2fcc659d13..0c90a9bdda 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-widget-select.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-widget-select.component.ts @@ -14,23 +14,12 @@ /// limitations under the License. /// -import { - Component, - OnDestroy, - OnInit, - ViewEncapsulation, - Input, - Output, - EventEmitter, - OnChanges, - SimpleChanges -} from '@angular/core'; -import { PageComponent } from '@shared/components/page.component'; +import { Component, EventEmitter, Input, OnChanges, OnInit, Output, SimpleChanges } from '@angular/core'; import { WidgetsBundle } from '@shared/models/widgets-bundle.model'; import { IAliasController } from '@core/api/widget-api.models'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { WidgetService } from '@core/http/widget.service'; -import { widgetType, Widget } from '@shared/models/widget.models'; +import { Widget, widgetType } from '@shared/models/widget.models'; import { toWidgetInfo } from '@home/models/widget-component.models'; import { DashboardCallbacks } from '../../models/dashboard-component.models'; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.ts index 553d3515d0..38c49b9f16 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.ts @@ -14,35 +14,18 @@ /// limitations under the License. /// -import { Component, OnInit, Input, OnChanges, SimpleChanges, ViewChild, ChangeDetectionStrategy } from '@angular/core'; +import { Component, Input, OnChanges, OnInit, SimpleChanges, ViewChild } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { ActivatedRoute, Router } from '@angular/router'; -import { WidgetService } from '@core/http/widget.service'; -import { DialogService } from '@core/services/dialog.service'; import { MatDialog } from '@angular/material/dialog'; -import { TranslateService } from '@ngx-translate/core'; import { Dashboard, WidgetLayout } from '@shared/models/dashboard.models'; import { IAliasController } from '@core/api/widget-api.models'; -import { Widget, WidgetActionSource, WidgetTypeParameters } from '@shared/models/widget.models'; +import { Widget } from '@shared/models/widget.models'; import { WidgetComponentService } from '@home/components/widget/widget-component.service'; import { WidgetConfigComponentData } from '../../models/widget-component.models'; -import { deepClone, isDefined, isString } from '@core/utils'; -import { FormBuilder, FormGroup, NgForm, Validators } from '@angular/forms'; -import { EntityType } from '@shared/models/entity-type.models'; -import { Observable, of } from 'rxjs'; -import { EntityAlias, EntityAliases } from '@shared/models/alias.models'; -import { WidgetConfigCallbacks } from '@home/components/widget/widget-config.component.models'; -import { - EntityAliasesDialogComponent, - EntityAliasesDialogData -} from '@home/components/alias/entity-aliases-dialog.component'; -import { - EntityAliasDialogComponent, - EntityAliasDialogData -} from '@home/components/alias/entity-alias-dialog.component'; -import { tap } from 'rxjs/operators'; +import { isDefined, isString } from '@core/utils'; +import { FormBuilder, FormGroup, NgForm } from '@angular/forms'; @Component({ selector: 'tb-edit-widget', diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/make-dashboard-public-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/make-dashboard-public-dialog.component.ts index 92bfdf2d23..caceb2023b 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/make-dashboard-public-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/make-dashboard-public-dialog.component.ts @@ -14,19 +14,15 @@ /// limitations under the License. /// -import {Component, Inject, OnInit, SkipSelf} from '@angular/core'; -import { ErrorStateMatcher } from '@angular/material/core'; +import { Component, Inject, OnInit } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import {PageComponent} from '@shared/components/page.component'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm} from '@angular/forms'; -import {EntityType} from '@shared/models/entity-type.models'; -import {DashboardService} from '@core/http/dashboard.service'; -import {forkJoin, Observable} from 'rxjs'; -import {DashboardInfo} from '@app/shared/models/dashboard.models'; -import {ActionNotificationShow} from '@core/notification/notification.actions'; -import {TranslateService} from '@ngx-translate/core'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder } from '@angular/forms'; +import { DashboardService } from '@core/http/dashboard.service'; +import { DashboardInfo } from '@app/shared/models/dashboard.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; import { DialogComponent } from '@shared/components/dialog.component'; import { Router } from '@angular/router'; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.ts index e8950e8eab..b3ea169238 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.ts @@ -14,16 +14,15 @@ /// limitations under the License. /// -import {Component, Inject, OnInit, SkipSelf} from '@angular/core'; +import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; import { ErrorStateMatcher } from '@angular/material/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import {PageComponent} from '@shared/components/page.component'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm} from '@angular/forms'; -import {EntityType} from '@shared/models/entity-type.models'; -import {DashboardService} from '@core/http/dashboard.service'; -import {forkJoin, Observable} from 'rxjs'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm } from '@angular/forms'; +import { EntityType } from '@shared/models/entity-type.models'; +import { DashboardService } from '@core/http/dashboard.service'; +import { forkJoin, Observable } from 'rxjs'; import { DialogComponent } from '@shared/components/dialog.component'; import { Router } from '@angular/router'; @@ -122,13 +121,10 @@ export class ManageDashboardCustomersDialogComponent extends switch (this.data.actionType) { case 'assign': return this.dashboardService.addDashboardCustomers(dashboardId, customerIds); - break; case 'manage': return this.dashboardService.updateDashboardCustomers(dashboardId, customerIds); - break; case 'unassign': return this.dashboardService.removeDashboardCustomers(dashboardId, customerIds); - break; } } } diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/state-controller.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/states/state-controller.component.ts index f8deb4c035..b0825e8aa9 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/state-controller.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/state-controller.component.ts @@ -19,7 +19,7 @@ import { IDashboardController } from '../dashboard-page.models'; import { DashboardState } from '@app/shared/models/dashboard.models'; import { Subscription } from 'rxjs'; import { NgZone, OnDestroy, OnInit } from '@angular/core'; -import { ActivatedRoute, Router, Params } from '@angular/router'; +import { ActivatedRoute, Params, Router } from '@angular/router'; import { StatesControllerService } from '@home/pages/dashboard/states/states-controller.service'; import { EntityId } from '@app/shared/models/id/entity-id'; import { StateParams } from '@app/core/api/widget-api.models'; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-table-header.component.ts b/ui-ngx/src/app/modules/home/pages/device/device-table-header.component.ts index 10eff975a5..4137a6752d 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-table-header.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device-table-header.component.ts @@ -18,8 +18,8 @@ import { Component } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityTableHeaderComponent } from '../../components/entity/entity-table-header.component'; -import {DeviceInfo} from '@app/shared/models/device.models'; -import {EntityType} from '@shared/models/entity-type.models'; +import { DeviceInfo } from '@app/shared/models/device.models'; +import { EntityType } from '@shared/models/entity-type.models'; @Component({ selector: 'tb-device-table-header', diff --git a/ui-ngx/src/app/modules/home/pages/home-links/home-links-routing.module.ts b/ui-ngx/src/app/modules/home/pages/home-links/home-links-routing.module.ts index 6229d491f5..94fca64ac2 100644 --- a/ui-ngx/src/app/modules/home/pages/home-links/home-links-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-links/home-links-routing.module.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import {NgModule} from '@angular/core'; -import {RouterModule, Routes} from '@angular/router'; +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; -import {HomeLinksComponent} from './home-links.component'; -import {Authority} from '@shared/models/authority.enum'; +import { HomeLinksComponent } from './home-links.component'; +import { Authority } from '@shared/models/authority.enum'; const routes: Routes = [ { diff --git a/ui-ngx/src/app/modules/home/pages/home-links/home-links.component.ts b/ui-ngx/src/app/modules/home/pages/home-links/home-links.component.ts index 2a2aaf64f4..2c43b4d702 100644 --- a/ui-ngx/src/app/modules/home/pages/home-links/home-links.component.ts +++ b/ui-ngx/src/app/modules/home/pages/home-links/home-links.component.ts @@ -15,9 +15,6 @@ /// import { Component, OnInit } from '@angular/core'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { PageComponent } from '@shared/components/page.component'; import { MenuService } from '@core/services/menu.service'; import { BreakpointObserver, BreakpointState } from '@angular/cdk/layout'; import { MediaBreakpoints } from '@shared/models/constants'; diff --git a/ui-ngx/src/app/modules/home/pages/home-links/home-links.module.ts b/ui-ngx/src/app/modules/home/pages/home-links/home-links.module.ts index 18ec7018bc..af2f2bf6be 100644 --- a/ui-ngx/src/app/modules/home/pages/home-links/home-links.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-links/home-links.module.ts @@ -18,7 +18,7 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { HomeLinksRoutingModule } from './home-links-routing.module'; -import { HomeLinksComponent} from './home-links.component'; +import { HomeLinksComponent } from './home-links.component'; import { SharedModule } from '@app/shared/shared.module'; @NgModule({ diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.scss b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.scss index 768e76b5a9..8d034392c6 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.scss +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.scss @@ -114,7 +114,7 @@ overflow-x: hidden; overflow-y: auto; .mat-expansion-panel { - border-radius: 0px; + border-radius: 0; &:last-child { margin-bottom: 5px; } diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts index 1b5e34a0f5..cd305ed81a 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts @@ -40,10 +40,12 @@ import { DialogService } from '@core/services/dialog.service'; import { AuthService } from '@core/auth/auth.service'; import { ActivatedRoute, Router } from '@angular/router'; import { - inputNodeComponent, NodeConnectionInfo, + inputNodeComponent, + NodeConnectionInfo, ResolvedRuleChainMetaData, RuleChain, - RuleChainConnectionInfo, RuleChainImport, + RuleChainConnectionInfo, + RuleChainImport, RuleChainMetaData, ruleChainNodeComponent } from '@shared/models/rule-chain.models'; @@ -73,8 +75,8 @@ import { MatMenuTrigger } from '@angular/material/menu'; import { ItemBufferService, RuleNodeConnection } from '@core/services/item-buffer.service'; import { Hotkey } from 'angular2-hotkeys'; import { EntityType } from '@shared/models/entity-type.models'; -import Timeout = NodeJS.Timeout; import { DebugEventType, EventType } from '@shared/models/event.models'; +import Timeout = NodeJS.Timeout; @Component({ selector: 'tb-rulechain-page', diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-routing.module.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-routing.module.ts index d750559c00..99c8bf2a9f 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-routing.module.ts @@ -16,13 +16,16 @@ import * as AngularCore from '@angular/core'; import { Injectable, NgModule } from '@angular/core'; +import * as AngularRouter from '@angular/router'; import { ActivatedRouteSnapshot, CanActivate, - Resolve, Router, + Resolve, + Router, RouterModule, RouterStateSnapshot, - Routes, UrlTree + Routes, + UrlTree } from '@angular/router'; import { EntitiesTableComponent } from '../../components/entity/entities-table.component'; @@ -40,7 +43,6 @@ import { ConfirmOnExitGuard } from '@core/guards/confirm-on-exit.guard'; import * as AngularCommon from '@angular/common'; import * as AngularForms from '@angular/forms'; -import * as AngularRouter from '@angular/router'; import * as AngularCdkCoercion from '@angular/cdk/coercion'; import * as AngularCdkKeycodes from '@angular/cdk/keycodes'; import * as AngularMaterialChips from '@angular/material/chips'; @@ -49,10 +51,10 @@ import * as AngularMaterialDialog from '@angular/material/dialog'; import * as NgrxStore from '@ngrx/store'; import * as TranslateCore from '@ngx-translate/core'; import * as TbCore from '@core/public-api'; +import { ItemBufferService } from '@core/public-api'; import * as TbShared from '@shared/public-api'; import * as TbHomeComponents from '@home/components/public-api'; import * as _moment from 'moment'; -import { ItemBufferService } from '@core/public-api'; declare const SystemJS; diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.ts index 42acf01a3b..e658ed0d7e 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.ts @@ -18,7 +18,6 @@ import { Component } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityTabsComponent } from '../../components/entity/entity-tabs.component'; -import { AssetInfo } from '@app/shared/models/asset.models'; import { RuleChain } from '@shared/models/rule-chain.models'; @Component({ diff --git a/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.html b/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.html index 6bb9863d51..51c7236db2 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.html +++ b/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.html @@ -124,7 +124,7 @@ fxLayoutAlign="start center" *ngFor="let resource of widget.resources; let i = index" > + style="margin: 10px 0 0 0; max-height: 40px;"> diff --git a/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.ts b/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.ts index 9d6a172895..d931bbd697 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.ts +++ b/ui-ngx/src/app/modules/home/pages/widget/widget-editor.component.ts @@ -45,8 +45,8 @@ import { SaveWidgetTypeAsDialogResult } from '@home/pages/widget/save-widget-type-as-dialog.component'; import { Subscription } from 'rxjs'; -import Timeout = NodeJS.Timeout; import { ResizeObserver } from '@juggle/resize-observer'; +import Timeout = NodeJS.Timeout; // @dynamic @Component({ diff --git a/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts b/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts index c5f188dfe1..04fe7fdad7 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts +++ b/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts @@ -14,12 +14,12 @@ /// limitations under the License. /// -import {NgModule} from '@angular/core'; -import {CommonModule} from '@angular/common'; -import {SharedModule} from '@shared/shared.module'; -import {WidgetsBundleComponent} from '@modules/home/pages/widget/widgets-bundle.component'; -import {WidgetLibraryRoutingModule} from '@modules/home/pages/widget/widget-library-routing.module'; -import {HomeComponentsModule} from '@modules/home/components/home-components.module'; +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { WidgetsBundleComponent } from '@modules/home/pages/widget/widgets-bundle.component'; +import { WidgetLibraryRoutingModule } from '@modules/home/pages/widget/widget-library-routing.module'; +import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { WidgetLibraryComponent } from './widget-library.component'; import { WidgetEditorComponent } from '@home/pages/widget/widget-editor.component'; import { SelectWidgetTypeDialogComponent } from '@home/pages/widget/select-widget-type-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle.component.ts b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle.component.ts index 1e70668556..26db26b4bf 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle.component.ts +++ b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle.component.ts @@ -15,11 +15,11 @@ /// import { Component, Inject } from '@angular/core'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {EntityComponent} from '../../components/entity/entity.component'; -import {FormBuilder, FormGroup, Validators} from '@angular/forms'; -import {WidgetsBundle} from '@shared/models/widgets-bundle.model'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityComponent } from '../../components/entity/entity.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { WidgetsBundle } from '@shared/models/widgets-bundle.model'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @Component({ diff --git a/ui-ngx/src/app/modules/login/pages/login/login.component.ts b/ui-ngx/src/app/modules/login/pages/login/login.component.ts index 7e3f99239f..a53d83f839 100644 --- a/ui-ngx/src/app/modules/login/pages/login/login.component.ts +++ b/ui-ngx/src/app/modules/login/pages/login/login.component.ts @@ -22,8 +22,7 @@ import { PageComponent } from '@shared/components/page.component'; import { FormBuilder } from '@angular/forms'; import { HttpErrorResponse } from '@angular/common/http'; import { Constants } from '@shared/models/constants'; -import { Router } from '@angular/router'; -import { ActivatedRoute } from '@angular/router'; +import { ActivatedRoute, Router } from '@angular/router'; @Component({ selector: 'tb-login', diff --git a/ui-ngx/src/app/shared/animations/speed-dial-fab.animations.ts b/ui-ngx/src/app/shared/animations/speed-dial-fab.animations.ts index 2583789d3d..d3c3d0fcb6 100644 --- a/ui-ngx/src/app/shared/animations/speed-dial-fab.animations.ts +++ b/ui-ngx/src/app/shared/animations/speed-dial-fab.animations.ts @@ -14,16 +14,7 @@ /// limitations under the License. /// -import { - animate, - keyframes, - query, - stagger, - state, - style, - transition, - trigger -} from '@angular/animations'; +import { animate, keyframes, query, stagger, state, style, transition, trigger } from '@angular/animations'; export const speedDialFabAnimations = [ trigger('fabToggler', [ diff --git a/ui-ngx/src/app/shared/components/circular-progress.directive.ts b/ui-ngx/src/app/shared/components/circular-progress.directive.ts index c53267a7be..a7b0b7b690 100644 --- a/ui-ngx/src/app/shared/components/circular-progress.directive.ts +++ b/ui-ngx/src/app/shared/components/circular-progress.directive.ts @@ -14,9 +14,7 @@ /// limitations under the License. /// -import { Directive, ElementRef, ViewContainerRef, ComponentFactoryResolver, ComponentRef, Input } from '@angular/core'; -import { Overlay } from '@angular/cdk/overlay'; -import { MatProgressBar } from '@angular/material/progress-bar'; +import { ComponentFactoryResolver, ComponentRef, Directive, ElementRef, Input, ViewContainerRef } from '@angular/core'; import { MatSpinner } from '@angular/material/progress-spinner'; @Directive({ diff --git a/ui-ngx/src/app/shared/components/dialog/material-icons-dialog.component.scss b/ui-ngx/src/app/shared/components/dialog/material-icons-dialog.component.scss index a8e9d8725b..5c0520d6ab 100644 --- a/ui-ngx/src/app/shared/components/dialog/material-icons-dialog.component.scss +++ b/ui-ngx/src/app/shared/components/dialog/material-icons-dialog.component.scss @@ -32,7 +32,7 @@ padding: 16px; margin: 10px; border: solid 1px #ffa500; - border-radius: 0%; + border-radius: 0; line-height: 0; } } diff --git a/ui-ngx/src/app/shared/components/dialog/node-script-test-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/node-script-test-dialog.component.html index 795513890e..060a6836fc 100644 --- a/ui-ngx/src/app/shared/components/dialog/node-script-test-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/node-script-test-dialog.component.html @@ -37,7 +37,7 @@
diff --git a/ui-ngx/src/app/shared/components/dialog/todo-dialog.component.ts b/ui-ngx/src/app/shared/components/dialog/todo-dialog.component.ts index 482aa450a6..d641012d3a 100644 --- a/ui-ngx/src/app/shared/components/dialog/todo-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/dialog/todo-dialog.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Component} from '@angular/core'; +import { Component } from '@angular/core'; import { MatDialogRef } from '@angular/material/dialog'; @Component({ diff --git a/ui-ngx/src/app/shared/components/directives/tb-json-to-string.directive.ts b/ui-ngx/src/app/shared/components/directives/tb-json-to-string.directive.ts index f19022e590..bcd10135c0 100644 --- a/ui-ngx/src/app/shared/components/directives/tb-json-to-string.directive.ts +++ b/ui-ngx/src/app/shared/components/directives/tb-json-to-string.directive.ts @@ -14,16 +14,18 @@ /// limitations under the License. /// -import {Directive, ElementRef, forwardRef, HostListener, Renderer2, SkipSelf} from '@angular/core'; +import { Directive, ElementRef, forwardRef, HostListener, Renderer2, SkipSelf } from '@angular/core'; import { ControlValueAccessor, - FormControl, FormGroupDirective, + FormControl, + FormGroupDirective, NG_VALIDATORS, - NG_VALUE_ACCESSOR, NgForm, + NG_VALUE_ACCESSOR, + NgForm, ValidationErrors, Validator } from '@angular/forms'; -import {ErrorStateMatcher} from '@angular/material/core'; +import { ErrorStateMatcher } from '@angular/material/core'; @Directive({ selector: '[tb-json-to-string]', diff --git a/ui-ngx/src/app/shared/components/entity/entity-list-select.component.ts b/ui-ngx/src/app/shared/components/entity/entity-list-select.component.ts index 26e0e6b5f9..94c52bbcb1 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list-select.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-list-select.component.ts @@ -14,15 +14,15 @@ /// limitations under the License. /// -import {AfterViewInit, Component, forwardRef, Input, OnInit} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR} from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {TranslateService} from '@ngx-translate/core'; -import {AliasEntityType, EntityType, entityTypeTranslations} from '@shared/models/entity-type.models'; -import {EntityService} from '@core/http/entity.service'; -import {EntityId} from '@shared/models/id/entity-id'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { AfterViewInit, Component, forwardRef, Input, OnInit } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; +import { EntityService } from '@core/http/entity.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; interface EntityListSelectModel { entityType: EntityType | AliasEntityType; diff --git a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts index f8bdab6243..5595708b96 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts @@ -15,38 +15,29 @@ /// import { - AfterContentInit, AfterViewInit, Component, ElementRef, forwardRef, - Input, OnChanges, - OnInit, SimpleChanges, - SkipSelf, + Input, + OnChanges, + OnInit, + SimpleChanges, ViewChild } from '@angular/core'; -import { - ControlValueAccessor, - FormBuilder, - FormControl, - FormGroup, - FormGroupDirective, - NG_VALUE_ACCESSOR, NgForm, Validators -} from '@angular/forms'; -import {Observable, of} from 'rxjs'; -import {map, mergeMap, startWith, tap, share, pairwise, filter} from 'rxjs/operators'; -import {Store} from '@ngrx/store'; -import {AppState} from '@app/core/core.state'; -import {TranslateService} from '@ngx-translate/core'; -import {AliasEntityType, EntityType} from '@shared/models/entity-type.models'; -import {BaseData} from '@shared/models/base-data'; -import {EntityId} from '@shared/models/id/entity-id'; -import {EntityService} from '@core/http/entity.service'; -import { MatAutocomplete, MatAutocompleteSelectedEvent } from '@angular/material/autocomplete'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Observable } from 'rxjs'; +import { filter, map, mergeMap, share, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType } from '@shared/models/entity-type.models'; +import { BaseData } from '@shared/models/base-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityService } from '@core/http/entity.service'; +import { MatAutocomplete } from '@angular/material/autocomplete'; import { MatChipList } from '@angular/material/chips'; -import { ErrorStateMatcher } from '@angular/material/core'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { emptyPageData } from '@shared/models/page/page-data'; @Component({ selector: 'tb-entity-list', diff --git a/ui-ngx/src/app/shared/components/entity/entity-subtype-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-subtype-autocomplete.component.ts index 487a7520d9..59d7920106 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-subtype-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-subtype-autocomplete.component.ts @@ -14,27 +14,19 @@ /// limitations under the License. /// -import {AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild, OnDestroy} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR} from '@angular/forms'; -import {Observable, of, throwError, Subscription} from 'rxjs'; -import {PageLink} from '@shared/models/page/page-link'; -import {Direction} from '@shared/models/page/sort-order'; -import {filter, map, mergeMap, publishReplay, refCount, startWith, tap, publish} from 'rxjs/operators'; -import {PageData, emptyPageData} from '@shared/models/page/page-data'; -import {DashboardInfo} from '@app/shared/models/dashboard.models'; -import {DashboardId} from '@app/shared/models/id/dashboard-id'; -import {DashboardService} from '@core/http/dashboard.service'; -import {Store} from '@ngrx/store'; -import {AppState} from '@app/core/core.state'; -import {getCurrentAuthUser} from '@app/core/auth/auth.selectors'; -import {Authority} from '@shared/models/authority.enum'; -import {TranslateService} from '@ngx-translate/core'; -import {DeviceService} from '@core/http/device.service'; -import {EntitySubtype, EntityType} from '@app/shared/models/entity-type.models'; -import {BroadcastService} from '@app/core/services/broadcast.service'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; -import {AssetService} from '@core/http/asset.service'; -import {EntityViewService} from '@core/http/entity-view.service'; +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable, Subscription, throwError } from 'rxjs'; +import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { DeviceService } from '@core/http/device.service'; +import { EntitySubtype, EntityType } from '@app/shared/models/entity-type.models'; +import { BroadcastService } from '@app/core/services/broadcast.service'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { AssetService } from '@core/http/asset.service'; +import { EntityViewService } from '@core/http/entity-view.service'; @Component({ selector: 'tb-entity-subtype-autocomplete', diff --git a/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.ts b/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.ts index 981e9f407f..8ccd81edf5 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.ts @@ -14,44 +14,21 @@ /// limitations under the License. /// -import { - AfterContentInit, - AfterViewInit, - Component, - ElementRef, - forwardRef, - Input, OnDestroy, - OnInit, - SkipSelf, - ViewChild -} from '@angular/core'; -import { - ControlValueAccessor, - FormBuilder, - FormControl, - FormGroup, - FormGroupDirective, - NG_VALUE_ACCESSOR, NgForm, Validators -} from '@angular/forms'; -import { Observable, of, Subscription, throwError } from 'rxjs'; -import { map, mergeMap, startWith, tap, share, pairwise, filter, publishReplay, refCount } from 'rxjs/operators'; -import {Store} from '@ngrx/store'; -import {AppState} from '@app/core/core.state'; -import {TranslateService} from '@ngx-translate/core'; -import { AliasEntityType, EntitySubtype, EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; -import {BaseData} from '@shared/models/base-data'; -import {EntityId} from '@shared/models/id/entity-id'; -import {EntityService} from '@core/http/entity.service'; +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Observable, Subscription, throwError } from 'rxjs'; +import { map, mergeMap, publishReplay, refCount, share } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntitySubtype, EntityType } from '@shared/models/entity-type.models'; import { MatAutocomplete, MatAutocompleteSelectedEvent } from '@angular/material/autocomplete'; -import { MatChipList } from '@angular/material/chips'; -import { ErrorStateMatcher } from '@angular/material/core'; +import { MatChipInputEvent, MatChipList } from '@angular/material/chips'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { emptyPageData } from '@shared/models/page/page-data'; import { AssetService } from '@core/http/asset.service'; import { DeviceService } from '@core/http/device.service'; import { EntityViewService } from '@core/http/entity-view.service'; import { BroadcastService } from '@core/services/broadcast.service'; -import { MatChipInputEvent } from '@angular/material/chips'; import { COMMA, ENTER, SEMICOLON } from '@angular/cdk/keycodes'; @Component({ diff --git a/ui-ngx/src/app/shared/components/fab-toolbar.component.ts b/ui-ngx/src/app/shared/components/fab-toolbar.component.ts index 3799894eb5..e14b0b9cfb 100644 --- a/ui-ngx/src/app/shared/components/fab-toolbar.component.ts +++ b/ui-ngx/src/app/shared/components/fab-toolbar.component.ts @@ -143,7 +143,6 @@ export class FabToolbarComponent extends MatFabToolbarMixinBase implements OnIni const actions = element.find('mat-fab-actions').children(); if (triggerElement && backgroundElement) { const width = el.offsetWidth; - const height = el.offsetHeight; const scale = 2 * (width / triggerElement.offsetWidth); backgroundElement.style.borderRadius = width + 'px'; diff --git a/ui-ngx/src/app/shared/components/file-input.component.scss b/ui-ngx/src/app/shared/components/file-input.component.scss index c3ab44b653..f4085a87fc 100644 --- a/ui-ngx/src/app/shared/components/file-input.component.scss +++ b/ui-ngx/src/app/shared/components/file-input.component.scss @@ -20,7 +20,7 @@ $previewSize: 100px !default; :host { .tb-container { - margin-top: 0px; + margin-top: 0; label.tb-title { display: block; padding-bottom: 8px; diff --git a/ui-ngx/src/app/shared/components/file-input.component.ts b/ui-ngx/src/app/shared/components/file-input.component.ts index b305aa509e..56f6be1dd1 100644 --- a/ui-ngx/src/app/shared/components/file-input.component.ts +++ b/ui-ngx/src/app/shared/components/file-input.component.ts @@ -22,7 +22,8 @@ import { Input, OnChanges, OnDestroy, - Output, SimpleChanges, + Output, + SimpleChanges, ViewChild } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; diff --git a/ui-ngx/src/app/shared/components/footer-fab-buttons.component.scss b/ui-ngx/src/app/shared/components/footer-fab-buttons.component.scss index 916141aced..106692602b 100644 --- a/ui-ngx/src/app/shared/components/footer-fab-buttons.component.scss +++ b/ui-ngx/src/app/shared/components/footer-fab-buttons.component.scss @@ -40,7 +40,7 @@ .tb-btn-footer { &.fab-toggler { - margin-top: 0px; + margin-top: 0; } position: relative !important; display: inline-block !important; diff --git a/ui-ngx/src/app/shared/components/fullscreen.directive.ts b/ui-ngx/src/app/shared/components/fullscreen.directive.ts index 87214cc317..20558c7c75 100644 --- a/ui-ngx/src/app/shared/components/fullscreen.directive.ts +++ b/ui-ngx/src/app/shared/components/fullscreen.directive.ts @@ -18,8 +18,13 @@ import { Directive, ElementRef, EventEmitter, - Input, OnChanges, OnDestroy, - Output, Renderer2, SecurityContext, SimpleChanges, + Input, + OnChanges, + OnDestroy, + Output, + Renderer2, + SecurityContext, + SimpleChanges, ViewContainerRef } from '@angular/core'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; diff --git a/ui-ngx/src/app/shared/components/hotkeys.directive.ts b/ui-ngx/src/app/shared/components/hotkeys.directive.ts index f67d56280e..744a0765e6 100644 --- a/ui-ngx/src/app/shared/components/hotkeys.directive.ts +++ b/ui-ngx/src/app/shared/components/hotkeys.directive.ts @@ -14,8 +14,8 @@ /// limitations under the License. /// -import {Directive, Input, OnInit, OnDestroy, ElementRef} from '@angular/core'; -import {Hotkey, ExtendedKeyboardEvent} from 'angular2-hotkeys'; +import { Directive, ElementRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { Hotkey } from 'angular2-hotkeys'; import 'mousetrap'; import { TbCheatSheetComponent } from '@shared/components/cheatsheet.component'; diff --git a/ui-ngx/src/app/shared/components/image-input.component.scss b/ui-ngx/src/app/shared/components/image-input.component.scss index d4601f81ec..986c1253b3 100644 --- a/ui-ngx/src/app/shared/components/image-input.component.scss +++ b/ui-ngx/src/app/shared/components/image-input.component.scss @@ -20,7 +20,7 @@ $previewSize: 100px !default; :host { .tb-container { - margin-top: 0px; + margin-top: 0; label.tb-title { display: block; padding-bottom: 8px; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-date.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-date.tsx index 4a7f265ad8..b33829ca11 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-date.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-date.tsx @@ -16,10 +16,7 @@ import * as React from 'react'; import ThingsboardBaseComponent from './json-form-base-component'; import DateFnsUtils from '@date-io/date-fns'; -import { - MuiPickersUtilsProvider, - KeyboardDatePicker -} from '@material-ui/pickers'; +import { KeyboardDatePicker, MuiPickersUtilsProvider } from '@material-ui/pickers'; import { JsonFormFieldProps, JsonFormFieldState } from '@shared/components/json-form/react/json-form.models'; interface ThingsboardDateState extends JsonFormFieldState { diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-image.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-image.tsx index f7f01796d5..bfc407a61b 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-image.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-image.tsx @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, {useCallback} from 'react'; -import { DropzoneState, useDropzone } from 'react-dropzone'; -import ThingsboardBaseComponent from './json-form-base-component'; +import React from 'react'; import Dropzone from 'react-dropzone'; +import ThingsboardBaseComponent from './json-form-base-component'; import { JsonFormFieldProps, JsonFormFieldState } from '@shared/components/json-form/react/json-form.models'; import IconButton from '@material-ui/core/IconButton'; import ClearIcon from '@material-ui/icons/Clear'; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-radios.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-radios.tsx index 8b2789f9c9..8aa63aa4e4 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-radios.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-radios.tsx @@ -16,7 +16,7 @@ import * as React from 'react'; import { JsonFormFieldProps, JsonFormFieldState } from '@shared/components/json-form/react/json-form.models'; import FormControlLabel from '@material-ui/core/FormControlLabel'; -import { Radio, FormLabel, RadioGroup } from '@material-ui/core'; +import { FormLabel, Radio, RadioGroup } from '@material-ui/core'; import FormControl from '@material-ui/core/FormControl'; import ThingsboardBaseComponent from '@shared/components/json-form/react/json-form-base-component'; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-rc-select.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-rc-select.tsx index 842c897e36..a6473c7967 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-rc-select.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-rc-select.tsx @@ -15,7 +15,7 @@ */ import * as React from 'react'; import ThingsboardBaseComponent from './json-form-base-component'; -import Select, {Option} from 'rc-select'; +import Select, { Option } from 'rc-select'; import { JsonFormFieldProps, JsonFormFieldState, diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-schema-form.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-schema-form.tsx index 3fbfbe2a61..50d48565a0 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-schema-form.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-schema-form.tsx @@ -33,11 +33,12 @@ import ThingsboardCheckbox from './json-form-checkbox'; import ThingsboardHelp from './json-form-help'; import ThingsboardFieldSet from './json-form-fieldset'; import ThingsboardIcon from './json-form-icon'; -import { JsonFormProps, JsonFormData, onChangeFn, OnColorClickFn, OnIconClickFn } from './json-form.models'; +import { JsonFormData, JsonFormProps, onChangeFn, OnColorClickFn, OnIconClickFn } from './json-form.models'; import _ from 'lodash'; import * as tinycolor_ from 'tinycolor2'; import { GroupInfo } from '@shared/models/widget.models'; + const tinycolor = tinycolor_; class ThingsboardSchemaForm extends React.Component { diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-select.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-select.tsx index 3695df6fe4..0653209e6a 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-select.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-select.tsx @@ -14,11 +14,7 @@ * limitations under the License. */ import * as React from 'react'; -import { - JsonFormFieldProps, - JsonFormFieldState, - KeyLabelItem -} from '@shared/components/json-form/react/json-form.models'; +import { JsonFormFieldProps, JsonFormFieldState } from '@shared/components/json-form/react/json-form.models'; import MenuItem from '@material-ui/core/MenuItem'; import FormControl from '@material-ui/core/FormControl'; import InputLabel from '@material-ui/core/InputLabel'; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-text.tsx b/ui-ngx/src/app/shared/components/json-form/react/json-form-text.tsx index d043d9899e..0b690f8181 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-text.tsx +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-text.tsx @@ -17,7 +17,6 @@ import * as React from 'react'; import ThingsboardBaseComponent from './json-form-base-component'; import TextField from '@material-ui/core/TextField'; import { JsonFormFieldProps, JsonFormFieldState } from '@shared/components/json-form/react/json-form.models'; -import { FormControl } from '@material-ui/core'; interface ThingsboardTextState extends JsonFormFieldState { focused: boolean; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form-utils.ts b/ui-ngx/src/app/shared/components/json-form/react/json-form-utils.ts index b83b11d7d5..6937a4acb8 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form-utils.ts +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form-utils.ts @@ -18,11 +18,11 @@ import * as tv from 'tv4'; import ObjectPath from 'objectpath'; import _ from 'lodash'; import { - SchemaValidationResult, DefaultsFormOptions, FormOption, + JsonFormData, JsonSchemaData, - JsonFormData + SchemaValidationResult } from './json-form.models'; import { isDefined, isEqual, isString, isUndefined } from '@core/utils'; diff --git a/ui-ngx/src/app/shared/components/json-form/react/json-form.scss b/ui-ngx/src/app/shared/components/json-form/react/json-form.scss index fbe48114cc..a79b9d24d6 100644 --- a/ui-ngx/src/app/shared/components/json-form/react/json-form.scss +++ b/ui-ngx/src/app/shared/components/json-form/react/json-form.scss @@ -319,7 +319,7 @@ $previewSize: 100px !default; z-index: 100001; background-color: white; border: 1px solid #d9d9d9; - box-shadow: 0 0px 4px #d9d9d9; + box-shadow: 0 0 4px #d9d9d9; border-radius: 4px; box-sizing: border-box; outline: none; diff --git a/ui-ngx/src/app/shared/components/kv-map.component.ts b/ui-ngx/src/app/shared/components/kv-map.component.ts index a932e302e2..b6f438f428 100644 --- a/ui-ngx/src/app/shared/components/kv-map.component.ts +++ b/ui-ngx/src/app/shared/components/kv-map.component.ts @@ -14,21 +14,19 @@ /// limitations under the License. /// -import { Component, EventEmitter, forwardRef, Input, OnInit, Output } from '@angular/core'; +import { Component, forwardRef, Input, OnInit } from '@angular/core'; import { AbstractControl, - ControlValueAccessor, FormArray, - FormBuilder, FormControl, - FormGroup, NG_VALIDATORS, - NG_VALUE_ACCESSOR, Validator, - ValidatorFn, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, Validators } from '@angular/forms'; -import { AliasFilterType, aliasFilterTypeTranslationMap, EntityAliasFilter } from '@shared/models/alias.models'; -import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; -import { TranslateService } from '@ngx-translate/core'; -import { EntityService } from '@core/http/entity.service'; -import { EntitySearchDirection, entitySearchDirectionTranslations, EntityTypeFilter } from '@shared/models/relation.models'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; diff --git a/ui-ngx/src/app/shared/components/led-light.component.ts b/ui-ngx/src/app/shared/components/led-light.component.ts index 3db4cf8d52..aa2c373dfa 100644 --- a/ui-ngx/src/app/shared/components/led-light.component.ts +++ b/ui-ngx/src/app/shared/components/led-light.component.ts @@ -16,8 +16,7 @@ import { AfterViewInit, Component, ElementRef, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { RaphaelElement, RaphaelPaper, RaphaelSet } from 'raphael'; -import Raphael from 'raphael'; +import Raphael, { RaphaelElement, RaphaelPaper, RaphaelSet } from 'raphael'; import * as tinycolor_ from 'tinycolor2'; const tinycolor = tinycolor_; diff --git a/ui-ngx/src/app/shared/components/nav-tree.component.scss b/ui-ngx/src/app/shared/components/nav-tree.component.scss index b9d9dc7f72..ae318587e5 100644 --- a/ui-ngx/src/app/shared/components/nav-tree.component.scss +++ b/ui-ngx/src/app/shared/components/nav-tree.component.scss @@ -228,7 +228,6 @@ } .jstree-leaf > .jstree-ocl { - background: 0 0; background-position: -40px -120px; } diff --git a/ui-ngx/src/app/shared/components/snack-bar-component.scss b/ui-ngx/src/app/shared/components/snack-bar-component.scss index b694ae9dae..054937fb45 100644 --- a/ui-ngx/src/app/shared/components/snack-bar-component.scss +++ b/ui-ngx/src/app/shared/components/snack-bar-component.scss @@ -17,18 +17,18 @@ display: inline-block; pointer-events: all; .tb-toast { - box-shadow: 0px 3px 5px -1px rgba(0, 0, 0, 0.2), 0px 6px 10px 0px rgba(0, 0, 0, 0.14), 0px 1px 18px 0px rgba(0, 0, 0, 0.12); + box-shadow: 0 3px 5px -1px rgba(0, 0, 0, 0.2), 0 6px 10px 0 rgba(0, 0, 0, 0.14), 0 1px 18px 0 rgba(0, 0, 0, 0.12); color: #fff; font-size: 18px; border-radius: 4px; - padding: 0px 18px; + padding: 0 18px; margin: 8px; .toast-text { - padding: 0px 6px; + padding: 0 6px; width: 100%; } button { - margin: 6px 0px 6px 12px; + margin: 6px 0 6px 12px; } &.info-toast { background: #323232; diff --git a/ui-ngx/src/app/shared/components/tb-error.component.ts b/ui-ngx/src/app/shared/components/tb-error.component.ts index 99ddc55f1a..2a19f6cb0a 100644 --- a/ui-ngx/src/app/shared/components/tb-error.component.ts +++ b/ui-ngx/src/app/shared/components/tb-error.component.ts @@ -15,7 +15,7 @@ /// import { Component, Input } from '@angular/core'; -import { trigger, state, style, transition, animate } from '@angular/animations'; +import { animate, state, style, transition, trigger } from '@angular/animations'; @Component({ selector: 'tb-error', diff --git a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss index 0ca2a9e5b8..8d93f982d9 100644 --- a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss +++ b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.scss @@ -51,16 +51,6 @@ padding: 0 0 2px; margin: 2px; line-height: 24px; - - ng-mat-icon { - width: 24px; - height: 24px; - - svg { - width: inherit; - height: inherit; - } - } } } diff --git a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts index 252720033c..08fe07abc2 100644 --- a/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts +++ b/ui-ngx/src/app/shared/components/time/history-selector/history-selector.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, OnInit, OnChanges, Input, Output, EventEmitter, ChangeDetectorRef } from '@angular/core'; +import { ChangeDetectorRef, Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core'; import { interval } from 'rxjs'; import { filter } from 'rxjs/operators'; import { HistorySelectSettings } from '@app/modules/home/components/widget/lib/maps/map-models'; diff --git a/ui-ngx/src/app/shared/components/toast.directive.ts b/ui-ngx/src/app/shared/components/toast.directive.ts index beb77666df..a8b304a16f 100644 --- a/ui-ngx/src/app/shared/components/toast.directive.ts +++ b/ui-ngx/src/app/shared/components/toast.directive.ts @@ -15,12 +15,16 @@ /// import { - AfterViewInit, ApplicationRef, ChangeDetectorRef, + AfterViewInit, + ChangeDetectorRef, Component, Directive, ElementRef, - Inject, Input, NgZone, - OnDestroy, ViewChild, + Inject, + Input, + NgZone, + OnDestroy, + ViewChild, ViewContainerRef } from '@angular/core'; import { MAT_SNACK_BAR_DATA, MatSnackBar, MatSnackBarConfig, MatSnackBarRef } from '@angular/material/snack-bar'; @@ -30,8 +34,8 @@ import { Subscription } from 'rxjs'; import { NotificationService } from '@app/core/services/notification.service'; import { BreakpointObserver } from '@angular/cdk/layout'; import { MediaBreakpoints } from '@shared/models/constants'; -import Timeout = NodeJS.Timeout; import { MatButton } from '@angular/material/button'; +import Timeout = NodeJS.Timeout; @Directive({ selector: '[tb-toast]' diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index 754dfced25..b189d8f4b6 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -55,7 +55,7 @@
- + {{ (modelValue ? 'value.true' : 'value.false') | translate }}
diff --git a/ui-ngx/src/app/shared/components/widgets-bundle-select.component.ts b/ui-ngx/src/app/shared/components/widgets-bundle-select.component.ts index 7a2ca60c71..90952c1ddb 100644 --- a/ui-ngx/src/app/shared/components/widgets-bundle-select.component.ts +++ b/ui-ngx/src/app/shared/components/widgets-bundle-select.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnChanges, OnInit, ViewEncapsulation, SimpleChanges } from '@angular/core'; +import { Component, forwardRef, Input, OnChanges, OnInit, SimpleChanges, ViewEncapsulation } from '@angular/core'; import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms'; import { Observable } from 'rxjs'; import { share, tap } from 'rxjs/operators'; diff --git a/ui-ngx/src/app/shared/models/alias.models.ts b/ui-ngx/src/app/shared/models/alias.models.ts index 50d26f10a0..3009685321 100644 --- a/ui-ngx/src/app/shared/models/alias.models.ts +++ b/ui-ngx/src/app/shared/models/alias.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; +import { EntityType } from '@shared/models/entity-type.models'; import { EntityId } from '@shared/models/id/entity-id'; import { EntitySearchDirection, EntityTypeFilter } from '@shared/models/relation.models'; import { EntityInfo } from './entity.models'; diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index 21610eae76..ab952d26a5 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,11 +14,10 @@ /// limitations under the License. /// -import {BaseData} from '@shared/models/base-data'; -import {AssetId} from './id/asset-id'; -import {TenantId} from '@shared/models/id/tenant-id'; -import {CustomerId} from '@shared/models/id/customer-id'; -import {DeviceCredentialsId} from '@shared/models/id/device-credentials-id'; +import { BaseData } from '@shared/models/base-data'; +import { AssetId } from './id/asset-id'; +import { TenantId } from '@shared/models/id/tenant-id'; +import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; export interface Asset extends BaseData { diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 81b44e6f8a..f33ea0ff3b 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -16,7 +16,7 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; -import {TenantId} from './id/tenant-id'; +import { TenantId } from './id/tenant-id'; export interface Customer extends ContactBased { tenantId: TenantId; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 292a21ca9d..781471c22f 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -20,8 +20,7 @@ import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; import { Widget } from './widget.models'; import { Timewindow } from '@shared/models/time/time.models'; -import { EntityType } from '@shared/models/entity-type.models'; -import { EntityAlias, EntityAliases } from './alias.models'; +import { EntityAliases } from './alias.models'; export interface DashboardInfo extends BaseData { tenantId?: TenantId; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 7378fc430a..780f60904e 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import {BaseData} from '@shared/models/base-data'; -import {DeviceId} from './id/device-id'; -import {TenantId} from '@shared/models/id/tenant-id'; -import {CustomerId} from '@shared/models/id/customer-id'; -import {DeviceCredentialsId} from '@shared/models/id/device-credentials-id'; +import { BaseData } from '@shared/models/base-data'; +import { DeviceId } from './id/device-id'; +import { TenantId } from '@shared/models/id/tenant-id'; +import { CustomerId } from '@shared/models/id/customer-id'; +import { DeviceCredentialsId } from '@shared/models/id/device-credentials-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; export interface Device extends BaseData { diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index b206281bce..83b9bcc1fe 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {TenantId} from './id/tenant-id'; +import { TenantId } from './id/tenant-id'; import { BaseData, HasId } from '@shared/models/base-data'; /// diff --git a/ui-ngx/src/app/shared/models/entity-view.models.ts b/ui-ngx/src/app/shared/models/entity-view.models.ts index 49c0218121..758916fe4b 100644 --- a/ui-ngx/src/app/shared/models/entity-view.models.ts +++ b/ui-ngx/src/app/shared/models/entity-view.models.ts @@ -14,12 +14,11 @@ /// limitations under the License. /// -import {BaseData} from '@shared/models/base-data'; -import {AssetId} from './id/asset-id'; -import {TenantId} from '@shared/models/id/tenant-id'; -import {CustomerId} from '@shared/models/id/customer-id'; -import {EntityViewId} from '@shared/models/id/entity-view-id'; -import {EntityId} from '@shared/models/id/entity-id'; +import { BaseData } from '@shared/models/base-data'; +import { TenantId } from '@shared/models/id/tenant-id'; +import { CustomerId } from '@shared/models/id/customer-id'; +import { EntityViewId } from '@shared/models/id/entity-view-id'; +import { EntityId } from '@shared/models/id/entity-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; export interface AttributesEntityView { diff --git a/ui-ngx/src/app/shared/models/id/entity-id.ts b/ui-ngx/src/app/shared/models/id/entity-id.ts index 0a0f0515bc..6ead753066 100644 --- a/ui-ngx/src/app/shared/models/id/entity-id.ts +++ b/ui-ngx/src/app/shared/models/id/entity-id.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {AliasEntityType, EntityType} from '@shared/models/entity-type.models'; +import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; import { HasUUID } from '@shared/models/id/has-uuid'; export interface EntityId extends HasUUID { diff --git a/ui-ngx/src/app/shared/models/material.models.ts b/ui-ngx/src/app/shared/models/material.models.ts index 22f751c640..7016898edf 100644 --- a/ui-ngx/src/app/shared/models/material.models.ts +++ b/ui-ngx/src/app/shared/models/material.models.ts @@ -15,6 +15,7 @@ /// import * as tinycolor_ from 'tinycolor2'; + const tinycolor = tinycolor_; export interface MaterialColorItem { diff --git a/ui-ngx/src/app/shared/models/page/page-data.ts b/ui-ngx/src/app/shared/models/page/page-data.ts index d5cfc71a9c..02662c4348 100644 --- a/ui-ngx/src/app/shared/models/page/page-data.ts +++ b/ui-ngx/src/app/shared/models/page/page-data.ts @@ -14,9 +14,6 @@ /// limitations under the License. /// -import { PageLink } from '@shared/models/page/page-link'; -import { Direction, SortOrder } from '@shared/models/page/sort-order'; - export interface PageData { data: Array; totalPages: number; diff --git a/ui-ngx/src/app/shared/models/relation.models.ts b/ui-ngx/src/app/shared/models/relation.models.ts index 0cb4f20ca9..dfb0b1d671 100644 --- a/ui-ngx/src/app/shared/models/relation.models.ts +++ b/ui-ngx/src/app/shared/models/relation.models.ts @@ -16,7 +16,6 @@ import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; -import { ActionStatus } from '@shared/models/audit-log.models'; export const CONTAINS_TYPE = 'Contains'; export const MANAGES_TYPE = 'Manages'; diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 4f2e01ea81..65a176de0d 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -14,9 +14,8 @@ /// limitations under the License. /// -import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; -import {TenantId} from './id/tenant-id'; +import { TenantId } from './id/tenant-id'; export interface Tenant extends ContactBased { title: string; diff --git a/ui-ngx/src/app/shared/models/user.model.ts b/ui-ngx/src/app/shared/models/user.model.ts index 468ed3d8c5..9a64242702 100644 --- a/ui-ngx/src/app/shared/models/user.model.ts +++ b/ui-ngx/src/app/shared/models/user.model.ts @@ -18,7 +18,7 @@ import { BaseData } from './base-data'; import { UserId } from './id/user-id'; import { CustomerId } from './id/customer-id'; import { Authority } from './authority.enum'; -import {TenantId} from './id/tenant-id'; +import { TenantId } from './id/tenant-id'; export interface User extends BaseData { tenantId: TenantId; diff --git a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts index 576be20de4..0d91340ed3 100644 --- a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts +++ b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts @@ -14,9 +14,9 @@ /// limitations under the License. /// -import {BaseData} from '@shared/models/base-data'; -import {TenantId} from '@shared/models/id/tenant-id'; -import {WidgetsBundleId} from '@shared/models/id/widgets-bundle-id'; +import { BaseData } from '@shared/models/base-data'; +import { TenantId } from '@shared/models/id/tenant-id'; +import { WidgetsBundleId } from '@shared/models/id/widgets-bundle-id'; export interface WidgetsBundle extends BaseData { tenantId: TenantId; diff --git a/ui-ngx/src/app/shared/pipe/highlight.pipe.ts b/ui-ngx/src/app/shared/pipe/highlight.pipe.ts index 12c591939c..bcd158d146 100644 --- a/ui-ngx/src/app/shared/pipe/highlight.pipe.ts +++ b/ui-ngx/src/app/shared/pipe/highlight.pipe.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Pipe, PipeTransform} from '@angular/core'; +import { Pipe, PipeTransform } from '@angular/core'; @Pipe({ name: 'highlight' }) export class HighlightPipe implements PipeTransform { diff --git a/ui-ngx/src/app/shared/pipe/tbJson.pipe.ts b/ui-ngx/src/app/shared/pipe/tbJson.pipe.ts index 5a5fa684fc..42d3f5a74c 100644 --- a/ui-ngx/src/app/shared/pipe/tbJson.pipe.ts +++ b/ui-ngx/src/app/shared/pipe/tbJson.pipe.ts @@ -14,8 +14,8 @@ /// limitations under the License. /// -import {Pipe, PipeTransform} from '@angular/core'; -import {isObject, isNumber} from '@core/utils'; +import { Pipe, PipeTransform } from '@angular/core'; +import { isNumber, isObject } from '@core/utils'; @Pipe({name: 'tbJson'}) export class TbJsonPipe implements PipeTransform { diff --git a/ui-ngx/src/app/shared/pipe/template.pipe.ts b/ui-ngx/src/app/shared/pipe/template.pipe.ts index 2cce658adb..4fa02a69e5 100644 --- a/ui-ngx/src/app/shared/pipe/template.pipe.ts +++ b/ui-ngx/src/app/shared/pipe/template.pipe.ts @@ -15,7 +15,7 @@ /// import { Pipe, PipeTransform } from '@angular/core'; -import { parseTemplate, parseWithTranslation } from '@app/core/utils'; +import { parseWithTranslation } from '@app/core/utils'; @Pipe({ name: 'tbParseTemplate' }) export class TbTemplatePipe implements PipeTransform { diff --git a/ui-ngx/src/app/shared/pipe/truncate.pipe.ts b/ui-ngx/src/app/shared/pipe/truncate.pipe.ts index b6c505ba74..2830b6ce7f 100644 --- a/ui-ngx/src/app/shared/pipe/truncate.pipe.ts +++ b/ui-ngx/src/app/shared/pipe/truncate.pipe.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Pipe, PipeTransform} from '@angular/core'; +import { Pipe, PipeTransform } from '@angular/core'; import { isString } from '@core/utils'; @Pipe({ name: 'truncate' }) diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index e94c8a6641..4914873095 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -20,7 +20,7 @@ import { FooterComponent } from '@shared/components/footer.component'; import { LogoComponent } from '@shared/components/logo.component'; import { TbSnackBarComponent, ToastDirective } from '@shared/components/toast.directive'; import { BreadcrumbComponent } from '@shared/components/breadcrumb.component'; -import { NgxFlowModule, FlowInjectionToken } from '@flowjs/ngx-flow'; +import { FlowInjectionToken, NgxFlowModule } from '@flowjs/ngx-flow'; import { NgxFlowchartModule } from 'ngx-flowchart/dist/ngx-flowchart'; import Flow from '@flowjs/flow.js'; @@ -94,7 +94,11 @@ import { EntityListSelectComponent } from '@shared/components/entity/entity-list import { JsonObjectEditComponent } from '@shared/components/json-object-edit.component'; import { FooterFabButtonsComponent } from '@shared/components/footer-fab-buttons.component'; import { CircularProgressDirective } from '@shared/components/circular-progress.directive'; -import { FabToolbarComponent, FabActionsDirective, FabTriggerDirective } from '@shared/components/fab-toolbar.component'; +import { + FabActionsDirective, + FabToolbarComponent, + FabTriggerDirective +} from '@shared/components/fab-toolbar.component'; import { DashboardSelectPanelComponent } from '@shared/components/dashboard-select-panel.component'; import { DashboardSelectComponent } from '@shared/components/dashboard-select.component'; import { WidgetsBundleSelectComponent } from '@shared/components/widgets-bundle-select.component'; diff --git a/ui-ngx/src/styles.scss b/ui-ngx/src/styles.scss index 61e9326286..ba963eb075 100644 --- a/ui-ngx/src/styles.scss +++ b/ui-ngx/src/styles.scss @@ -527,11 +527,10 @@ mat-label { .mat-table { width: 100%; max-width: 100%; - margin-bottom: 1rem; display: table; table-layout: auto; border-collapse: separate; - margin: 0px; + margin: 0; } mat-footer-row::after, mat-header-row::after, mat-row::after { From 77c6babff6662cf462cbce25cfc834daf352dafa Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 27 Apr 2020 09:43:33 +0300 Subject: [PATCH 098/102] Fix tenant isolated checkbox fields --- .../modules/home/pages/tenant/tenant.component.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts index f624a5d5cf..8a76830093 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts @@ -69,6 +69,20 @@ export class TenantComponent extends ContactBasedComponent { this.entityForm.patchValue({additionalInfo: {description: entity.additionalInfo ? entity.additionalInfo.description : ''}}); } + updateFormState() { + if (this.entityForm) { + if (this.isEditValue) { + this.entityForm.enable({emitEvent: false}); + if (!this.isAdd) { + this.entityForm.get('isolatedTbCore').disable({emitEvent: false}); + this.entityForm.get('isolatedTbRuleEngine').disable({emitEvent: false}); + } + } else { + this.entityForm.disable({emitEvent: false}); + } + } + } + onTenantIdCopied(event) { this.store.dispatch(new ActionNotificationShow( { From 3d71a5ba0ec0e171889734d9f8c92553a68f414d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 27 Apr 2020 09:46:39 +0300 Subject: [PATCH 099/102] Update rule nodes ui --- .../resources/public/static/rulenode/rulenode-core-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index d62d9f31da..1d13de3d18 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -13,4 +13,4 @@ See the Apache Version 2.0 License for specific language governing permissions and limitations under the License. ***************************************************************************** */var g=function(e,t){return(g=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var r in t)t.hasOwnProperty(r)&&(e[r]=t[r])})(e,t)};function y(e,t){function r(){this.constructor=e}g(e,t),e.prototype=null===t?Object.create(t):(r.prototype=t.prototype,new r)}function b(e,t,r,n){var a,o=arguments.length,i=o<3?t:null===n?n=Object.getOwnPropertyDescriptor(t,r):n;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)i=Reflect.decorate(e,t,r,n);else for(var l=e.length-1;l>=0;l--)(a=e[l])&&(i=(o<3?a(i):o>3?a(t,r,i):a(t,r))||i);return o>3&&i&&Object.defineProperty(t,r,i),i}function h(e,t){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(e,t)}function C(e){var t="function"==typeof Symbol&&Symbol.iterator,r=t&&e[t],n=0;if(r)return r.call(e);if(e&&"number"==typeof e.length)return{next:function(){return e&&n>=e.length&&(e=void 0),{value:e&&e[n++],done:!e}}};throw new TypeError(t?"Object is not iterable.":"Symbol.iterator is not defined.")}var v,F=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.emptyConfigForm},r.prototype.onConfigurationSet=function(e){this.emptyConfigForm=this.fb.group({})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-node-empty-config",template:"
"}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),T=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.attributeScopes=Object.keys(a.AttributeScope),n.telemetryTypeTranslationsMap=a.telemetryTypeTranslations,n}return y(r,e),r.prototype.configForm=function(){return this.attributesConfigForm},r.prototype.onConfigurationSet=function(e){this.attributesConfigForm=this.fb.group({scope:[e?e.scope:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-attributes-config",template:'
\n \n attribute.attributes-scope\n \n \n {{ telemetryTypeTranslationsMap.get(scope) | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),x=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.timeseriesConfigForm},r.prototype.onConfigurationSet=function(e){this.timeseriesConfigForm=this.fb.group({defaultTTL:[e?e.defaultTTL:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-timeseries-config",template:'
\n \n tb.rulenode.default-ttl\n \n \n {{ \'tb.rulenode.default-ttl-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-default-ttl-message\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),q=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.rpcRequestConfigForm},r.prototype.onConfigurationSet=function(e){this.rpcRequestConfigForm=this.fb.group({timeoutInSeconds:[e?e.timeoutInSeconds:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rpc-request-config",template:'
\n \n tb.rulenode.timeout-sec\n \n \n {{ \'tb.rulenode.timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-timeout-message\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),S=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.logConfigForm},r.prototype.onConfigurationSet=function(e){this.logConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.logConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"string",this.translate.instant("tb.rulenode.to-string"),"ToString",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.logConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-log-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),I=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.assignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.assignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],createCustomerIfNotExists:[!!e&&e.createCustomerIfNotExists,[]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n {{ \'tb.rulenode.create-customer-if-not-exists\' | translate }}\n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),k=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.clearAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.clearAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],alarmType:[e?e.alarmType:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.clearAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.clearAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-clear-alarm-config",template:'
\n \n \n \n
\n \n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),N=function(e){function r(t,r,n,o){var i=e.call(this,t)||this;return i.store=t,i.fb=r,i.nodeScriptTestService=n,i.translate=o,i.alarmSeverities=Object.keys(a.AlarmSeverity),i.alarmSeverityTranslationMap=a.alarmSeverityTranslations,i.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],i}return y(r,e),r.prototype.configForm=function(){return this.createAlarmConfigForm},r.prototype.onConfigurationSet=function(e){this.createAlarmConfigForm=this.fb.group({alarmDetailsBuildJs:[e?e.alarmDetailsBuildJs:null,[i.Validators.required]],useMessageAlarmData:[!!e&&e.useMessageAlarmData,[]],alarmType:[e?e.alarmType:null,[]],severity:[e?e.severity:null,[]],propagate:[!!e&&e.propagate,[]],relationTypes:[e?e.relationTypes:null,[]]})},r.prototype.validatorTriggers=function(){return["useMessageAlarmData"]},r.prototype.updateValidators=function(e){this.createAlarmConfigForm.get("useMessageAlarmData").value?(this.createAlarmConfigForm.get("alarmType").setValidators([]),this.createAlarmConfigForm.get("severity").setValidators([])):(this.createAlarmConfigForm.get("alarmType").setValidators([i.Validators.required]),this.createAlarmConfigForm.get("severity").setValidators([i.Validators.required])),this.createAlarmConfigForm.get("alarmType").updateValueAndValidity({emitEvent:e}),this.createAlarmConfigForm.get("severity").updateValueAndValidity({emitEvent:e})},r.prototype.testScript=function(){var e=this,t=this.createAlarmConfigForm.get("alarmDetailsBuildJs").value;this.nodeScriptTestService.testNodeScript(t,"json",this.translate.instant("tb.rulenode.details"),"Details",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.createAlarmConfigForm.get("alarmDetailsBuildJs").setValue(t)}))},r.prototype.removeKey=function(e,t){var r=this.createAlarmConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.createAlarmConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.createAlarmConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.createAlarmConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-create-alarm-config",template:'
\n \n \n \n
\n \n
\n \n {{ \'tb.rulenode.use-message-alarm-data\' | translate }}\n \n
\n
\n \n tb.rulenode.alarm-type\n \n \n {{ \'tb.rulenode.alarm-type-required\' | translate }}\n \n \n \n \n tb.rulenode.alarm-severity\n \n \n {{ alarmSeverityTranslationMap.get(severity) | translate }}\n \n \n \n {{ \'tb.rulenode.alarm-severity-required\' | translate }}\n \n \n
\n \n {{ \'tb.rulenode.propagate\' | translate }}\n \n
\n \n tb.rulenode.relation-types-list\n \n \n {{key}}\n close\n \n \n \n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),V=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.createRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.createRelationConfigForm=this.fb.group({direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[i.Validators.required]],entityNamePattern:[e?e.entityNamePattern:null,[]],entityTypePattern:[e?e.entityTypePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],createEntityIfNotExists:[!!e&&e.createEntityIfNotExists,[]],removeCurrentRelations:[!!e&&e.removeCurrentRelations,[]],changeOriginatorToRelatedEntity:[!!e&&e.changeOriginatorToRelatedEntity,[]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["entityType"]},r.prototype.updateValidators=function(e){var t=this.createRelationConfigForm.get("entityType").value;t?this.createRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.createRelationConfigForm.get("entityNamePattern").setValidators([]),!t||t!==a.EntityType.DEVICE&&t!==a.EntityType.ASSET?this.createRelationConfigForm.get("entityTypePattern").setValidators([]):this.createRelationConfigForm.get("entityTypePattern").setValidators([i.Validators.required]),this.createRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e}),this.createRelationConfigForm.get("entityTypePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-create-relation-config",template:'
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-type-pattern\n \n \n {{ \'tb.rulenode.entity-type-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n
\n \n {{ \'tb.rulenode.create-entity-if-not-exists\' | translate }}\n \n
tb.rulenode.create-entity-if-not-exists-hint
\n
\n \n {{ \'tb.rulenode.remove-current-relations\' | translate }}\n \n
tb.rulenode.remove-current-relations-hint
\n \n {{ \'tb.rulenode.change-originator-to-related-entity\' | translate }}\n \n
tb.rulenode.change-originator-to-related-entity-hint
\n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),E=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgDelayConfigForm},r.prototype.onConfigurationSet=function(e){this.msgDelayConfigForm=this.fb.group({useMetadataPeriodInSecondsPatterns:[!!e&&e.useMetadataPeriodInSecondsPatterns,[]],periodInSeconds:[e?e.periodInSeconds:null,[]],periodInSecondsPattern:[e?e.periodInSecondsPattern:null,[]],maxPendingMsgs:[e?e.maxPendingMsgs:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(1e5)]]})},r.prototype.validatorTriggers=function(){return["useMetadataPeriodInSecondsPatterns"]},r.prototype.updateValidators=function(e){this.msgDelayConfigForm.get("useMetadataPeriodInSecondsPatterns").value?(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([i.Validators.required]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([])):(this.msgDelayConfigForm.get("periodInSecondsPattern").setValidators([]),this.msgDelayConfigForm.get("periodInSeconds").setValidators([i.Validators.required,i.Validators.min(0)])),this.msgDelayConfigForm.get("periodInSecondsPattern").updateValueAndValidity({emitEvent:e}),this.msgDelayConfigForm.get("periodInSeconds").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-delay-config",template:'
\n \n {{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
\n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-0-seconds-message\' | translate }}\n \n \n \n \n tb.rulenode.period-in-seconds-pattern\n \n \n {{ \'tb.rulenode.period-in-seconds-pattern-required\' | translate }}\n \n \n \n \n \n tb.rulenode.max-pending-messages\n \n \n {{ \'tb.rulenode.max-pending-messages-required\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n {{ \'tb.rulenode.max-pending-messages-range\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),A=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n}return y(r,e),r.prototype.configForm=function(){return this.deleteRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.deleteRelationConfigForm=this.fb.group({deleteForSingleEntity:[!!e&&e.deleteForSingleEntity,[]],direction:[e?e.direction:null,[i.Validators.required]],entityType:[e?e.entityType:null,[]],entityNamePattern:[e?e.entityNamePattern:null,[]],relationType:[e?e.relationType:null,[i.Validators.required]],entityCacheExpiration:[e?e.entityCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.prototype.validatorTriggers=function(){return["deleteForSingleEntity","entityType"]},r.prototype.updateValidators=function(e){var t=this.deleteRelationConfigForm.get("deleteForSingleEntity").value,r=this.deleteRelationConfigForm.get("entityType").value;t?this.deleteRelationConfigForm.get("entityType").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityType").setValidators([]),t&&r?this.deleteRelationConfigForm.get("entityNamePattern").setValidators([i.Validators.required]):this.deleteRelationConfigForm.get("entityNamePattern").setValidators([]),this.deleteRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:!1}),this.deleteRelationConfigForm.get("entityNamePattern").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-delete-relation-config",template:'
\n \n {{ \'tb.rulenode.delete-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.delete-relation-hint
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n
\n \n \n \n tb.rulenode.entity-name-pattern\n \n \n {{ \'tb.rulenode.entity-name-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.relation-type-pattern\n \n \n {{ \'tb.rulenode.relation-type-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.entity-cache-expiration\n \n \n {{ \'tb.rulenode.entity-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.entity-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),L=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.generatorConfigForm},r.prototype.onConfigurationSet=function(e){this.generatorConfigForm=this.fb.group({msgCount:[e?e.msgCount:null,[i.Validators.required,i.Validators.min(0)]],periodInSeconds:[e?e.periodInSeconds:null,[i.Validators.required,i.Validators.min(1)]],originator:[e?e.originator:null,[]],jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.prepareInputConfig=function(e){return e&&(e.originatorId&&e.originatorType?e.originator={id:e.originatorId,entityType:e.originatorType}:e.originator=null,delete e.originatorId,delete e.originatorType),e},r.prototype.prepareOutputConfig=function(e){return e.originator?(e.originatorId=e.originator.id,e.originatorType=e.originator.entityType):(e.originatorId=null,e.originatorType=null),delete e.originator,e},r.prototype.testScript=function(){var e=this,t=this.generatorConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"generate",this.translate.instant("tb.rulenode.generator"),"Generate",["prevMsg","prevMetadata","prevMsgType"],this.ruleNodeId).subscribe((function(t){t&&e.generatorConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-action-node-generator-config",template:'
\n \n tb.rulenode.message-count\n \n \n {{ \'tb.rulenode.message-count-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-message-count-message\' | translate }}\n \n \n \n tb.rulenode.period-seconds\n \n \n {{ \'tb.rulenode.period-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-period-seconds-message\' | translate }}\n \n \n
\n \n \n \n
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent);!function(e){e.CUSTOMER="CUSTOMER",e.TENANT="TENANT",e.RELATED="RELATED",e.ALARM_ORIGINATOR="ALARM_ORIGINATOR"}(v||(v={}));var M,P=new Map([[v.CUSTOMER,"tb.rulenode.originator-customer"],[v.TENANT,"tb.rulenode.originator-tenant"],[v.RELATED,"tb.rulenode.originator-related"],[v.ALARM_ORIGINATOR,"tb.rulenode.originator-alarm-originator"]]);!function(e){e.CIRCLE="CIRCLE",e.POLYGON="POLYGON"}(M||(M={}));var R,w=new Map([[M.CIRCLE,"tb.rulenode.perimeter-circle"],[M.POLYGON,"tb.rulenode.perimeter-polygon"]]);!function(e){e.MILLISECONDS="MILLISECONDS",e.SECONDS="SECONDS",e.MINUTES="MINUTES",e.HOURS="HOURS",e.DAYS="DAYS"}(R||(R={}));var O,D=new Map([[R.MILLISECONDS,"tb.rulenode.time-unit-milliseconds"],[R.SECONDS,"tb.rulenode.time-unit-seconds"],[R.MINUTES,"tb.rulenode.time-unit-minutes"],[R.HOURS,"tb.rulenode.time-unit-hours"],[R.DAYS,"tb.rulenode.time-unit-days"]]);!function(e){e.METER="METER",e.KILOMETER="KILOMETER",e.FOOT="FOOT",e.MILE="MILE",e.NAUTICAL_MILE="NAUTICAL_MILE"}(O||(O={}));var K,B=new Map([[O.METER,"tb.rulenode.range-unit-meter"],[O.KILOMETER,"tb.rulenode.range-unit-kilometer"],[O.FOOT,"tb.rulenode.range-unit-foot"],[O.MILE,"tb.rulenode.range-unit-mile"],[O.NAUTICAL_MILE,"tb.rulenode.range-unit-nautical-mile"]]);!function(e){e.TITLE="TITLE",e.COUNTRY="COUNTRY",e.STATE="STATE",e.ZIP="ZIP",e.ADDRESS="ADDRESS",e.ADDRESS2="ADDRESS2",e.PHONE="PHONE",e.EMAIL="EMAIL",e.ADDITIONAL_INFO="ADDITIONAL_INFO"}(K||(K={}));var j,U,G,H=new Map([[K.TITLE,"tb.rulenode.entity-details-title"],[K.COUNTRY,"tb.rulenode.entity-details-country"],[K.STATE,"tb.rulenode.entity-details-state"],[K.ZIP,"tb.rulenode.entity-details-zip"],[K.ADDRESS,"tb.rulenode.entity-details-address"],[K.ADDRESS2,"tb.rulenode.entity-details-address2"],[K.PHONE,"tb.rulenode.entity-details-phone"],[K.EMAIL,"tb.rulenode.entity-details-email"],[K.ADDITIONAL_INFO,"tb.rulenode.entity-details-additional_info"]]);!function(e){e.FIRST="FIRST",e.LAST="LAST",e.ALL="ALL"}(j||(j={})),function(e){e.ASC="ASC",e.DESC="DESC"}(U||(U={})),function(e){e.STANDARD="STANDARD",e.FIFO="FIFO"}(G||(G={}));var Q,z=new Map([[G.STANDARD,"tb.rulenode.sqs-queue-standard"],[G.FIFO,"tb.rulenode.sqs-queue-fifo"]]),$=["anonymous","basic","cert.PEM"],_=new Map([["anonymous","tb.rulenode.credentials-anonymous"],["basic","tb.rulenode.credentials-basic"],["cert.PEM","tb.rulenode.credentials-pem"]]);!function(e){e.GET="GET",e.POST="POST",e.PUT="PUT",e.DELETE="DELETE"}(Q||(Q={}));var W=["US-ASCII","ISO-8859-1","UTF-8","UTF-16BE","UTF-16LE","UTF-16"],J=new Map([["US-ASCII","tb.rulenode.charset-us-ascii"],["ISO-8859-1","tb.rulenode.charset-iso-8859-1"],["UTF-8","tb.rulenode.charset-utf-8"],["UTF-16BE","tb.rulenode.charset-utf-16be"],["UTF-16LE","tb.rulenode.charset-utf-16le"],["UTF-16","tb.rulenode.charset-utf-16"]]),Y=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(O),n.rangeUnitTranslationMap=B,n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=D,n}return y(r,e),r.prototype.configForm=function(){return this.geoActionConfigForm},r.prototype.onConfigurationSet=function(e){this.geoActionConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]],minInsideDuration:[e?e.minInsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minInsideDurationTimeUnit:[e?e.minInsideDurationTimeUnit:null,[i.Validators.required]],minOutsideDuration:[e?e.minOutsideDuration:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]],minOutsideDurationTimeUnit:[e?e.minOutsideDurationTimeUnit:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoActionConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoActionConfigForm.get("perimeterType").value;t?this.geoActionConfigForm.get("perimeterType").setValidators([]):this.geoActionConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoActionConfigForm.get("centerLatitude").setValidators([]),this.geoActionConfigForm.get("centerLongitude").setValidators([]),this.geoActionConfigForm.get("range").setValidators([]),this.geoActionConfigForm.get("rangeUnit").setValidators([])):(this.geoActionConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoActionConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoActionConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoActionConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoActionConfigForm.get("polygonsDefinition").setValidators([]):this.geoActionConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoActionConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoActionConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoActionConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.min-inside-duration\n \n \n {{ \'tb.rulenode.min-inside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-inside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.min-outside-duration\n \n \n {{ \'tb.rulenode.min-outside-duration-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.min-outside-duration-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Z=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.msgCountConfigForm},r.prototype.onConfigurationSet=function(e){this.msgCountConfigForm=this.fb.group({interval:[e?e.interval:null,[i.Validators.required,i.Validators.min(1)]],telemetryPrefix:[e?e.telemetryPrefix:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-msg-count-config",template:'
\n \n tb.rulenode.interval-seconds\n \n \n {{ \'tb.rulenode.interval-seconds-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-interval-seconds-message\' | translate }}\n \n \n \n tb.rulenode.output-timeseries-key-prefix\n \n \n {{ \'tb.rulenode.output-timeseries-key-prefix-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),X=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.rpcReplyConfigForm},r.prototype.onConfigurationSet=function(e){this.rpcReplyConfigForm=this.fb.group({requestIdMetaDataAttribute:[e?e.requestIdMetaDataAttribute:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rpc-reply-config",template:'
\n \n tb.rulenode.request-id-metadata-attribute\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.saveToCustomTableConfigForm},r.prototype.onConfigurationSet=function(e){this.saveToCustomTableConfigForm=this.fb.group({tableName:[e?e.tableName:null,[i.Validators.required]],fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-custom-table-config",template:'
\n \n tb.rulenode.custom-table-name\n \n \n {{ \'tb.rulenode.custom-table-name-required\' | translate }}\n \n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),te=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.translate=r,o.injector=n,o.fb=a,o.propagateChange=null,o.valueChangeSubscription=null,o}var a;return y(r,e),a=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){this.ngControl=this.injector.get(i.NgControl),null!=this.ngControl&&(this.ngControl.valueAccessor=this),this.kvListFormGroup=this.fb.group({}),this.kvListFormGroup.addControl("keyVals",this.fb.array([]))},r.prototype.keyValsFormArray=function(){return this.kvListFormGroup.get("keyVals")},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.kvListFormGroup.disable({emitEvent:!1}):this.kvListFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t,r,n=this;this.valueChangeSubscription&&this.valueChangeSubscription.unsubscribe();var a=[];if(e)try{for(var o=C(Object.keys(e)),l=o.next();!l.done;l=o.next()){var s=l.value;Object.prototype.hasOwnProperty.call(e,s)&&a.push(this.fb.group({key:[s,[i.Validators.required]],value:[e[s],[i.Validators.required]]}))}}catch(e){t={error:e}}finally{try{l&&!l.done&&(r=o.return)&&r.call(o)}finally{if(t)throw t.error}}this.kvListFormGroup.setControl("keyVals",this.fb.array(a)),this.valueChangeSubscription=this.kvListFormGroup.valueChanges.subscribe((function(){n.updateModel()}))},r.prototype.removeKeyVal=function(e){this.kvListFormGroup.get("keyVals").removeAt(e)},r.prototype.addKeyVal=function(){this.kvListFormGroup.get("keyVals").push(this.fb.group({key:["",[i.Validators.required]],value:["",[i.Validators.required]]}))},r.prototype.validate=function(e){return!this.kvListFormGroup.get("keyVals").value.length&&this.required?{kvMapRequired:!0}:this.kvListFormGroup.valid?null:{kvFieldsRequired:!0}},r.prototype.updateModel=function(){var e=this.kvListFormGroup.get("keyVals").value;if(this.required&&!e.length||!this.kvListFormGroup.valid)this.propagateChange(null);else{var t={};e.forEach((function(e){t[e.key]=e.value})),this.propagateChange(t)}},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:t.Injector},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",String)],r.prototype,"requiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"keyRequiredText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valText",void 0),b([t.Input(),h("design:type",String)],r.prototype,"valRequiredText",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=a=b([t.Component({selector:"tb-kv-map-config",template:'
\n
\n {{ keyText }}\n {{ valText }}\n \n
\n
\n
\n \n \n \n \n {{ keyRequiredText | translate }}\n \n \n \n \n \n \n {{ valRequiredText | translate }}\n \n \n \n
\n
\n \n
\n \n
\n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return a})),multi:!0},{provide:i.NG_VALIDATORS,useExisting:t.forwardRef((function(){return a})),multi:!0}],styles:[":host .tb-kv-map-config{margin-bottom:16px}:host .tb-kv-map-config .header{padding-left:5px;padding-right:5px;padding-bottom:5px}:host .tb-kv-map-config .header .cell{padding-left:5px;padding-right:5px;color:rgba(0,0,0,.54);font-size:12px;font-weight:700;white-space:nowrap}:host .tb-kv-map-config .body{padding-left:5px;padding-right:5px;padding-bottom:20px;max-height:300px;overflow:auto}:host .tb-kv-map-config .body .row{padding-top:5px;max-height:40px}:host .tb-kv-map-config .body .cell{padding-left:5px;padding-right:5px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell{margin:0;max-height:40px}:host ::ng-deep .tb-kv-map-config .body mat-form-field.cell .mat-form-field-infix{border-top:0}:host ::ng-deep .tb-kv-map-config .body button.mat-button{margin:0}"]}),h("design:paramtypes",[o.Store,n.TranslateService,t.Injector,i.FormBuilder])],r)}(a.PageComponent),re=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.entityType=a.EntityType,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.deviceRelationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],relationType:[null],deviceTypes:[null,[i.Validators.required]]}),this.deviceRelationsQueryFormGroup.valueChanges.subscribe((function(t){e.deviceRelationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.deviceRelationsQueryFormGroup.disable({emitEvent:!1}):this.deviceRelationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.deviceRelationsQueryFormGroup.reset(e,{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-device-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-type
\n \n \n
device.device-types
\n \n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),ne=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.directionTypes=Object.keys(a.EntitySearchDirection),n.directionTypeTranslations=a.entitySearchDirectionTranslations,n.propagateChange=null,n}var n;return y(r,e),n=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.ngOnInit=function(){var e=this;this.relationsQueryFormGroup=this.fb.group({fetchLastLevelOnly:[!1,[]],direction:[null,[i.Validators.required]],maxLevel:[null,[]],filters:[null]}),this.relationsQueryFormGroup.valueChanges.subscribe((function(t){e.relationsQueryFormGroup.valid?e.propagateChange(t):e.propagateChange(null)}))},r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.relationsQueryFormGroup.disable({emitEvent:!1}):this.relationsQueryFormGroup.enable({emitEvent:!1})},r.prototype.writeValue=function(e){this.relationsQueryFormGroup.reset(e||{},{emitEvent:!1})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),r=n=b([t.Component({selector:"tb-relations-query-config",template:'
\n \n {{ \'alias.last-level-relation\' | translate }}\n \n
\n \n relation.direction\n \n \n {{ directionTypeTranslations.get(type) | translate }}\n \n \n \n \n tb.rulenode.max-relation-level\n \n \n
\n
relation.relation-filters
\n \n
\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return n})),multi:!0}]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.PageComponent),ae=function(e){function r(t,r,n,o){var i,l,m=e.call(this,t)||this;m.store=t,m.translate=r,m.truncate=n,m.fb=o,m.placeholder="tb.rulenode.message-type",m.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],m.messageTypes=[],m.messageTypesList=[],m.searchText="",m.propagateChange=function(e){},m.messageTypeConfigForm=m.fb.group({messageType:[null]});try{for(var u=C(Object.keys(a.MessageType)),d=u.next();!d.done;d=u.next()){var p=d.value;m.messageTypesList.push({name:a.messageTypeNames.get(a.MessageType[p]),value:p})}}catch(e){i={error:e}}finally{try{d&&!d.done&&(l=u.return)&&l.call(u)}finally{if(i)throw i.error}}return m}var l;return y(r,e),l=r,Object.defineProperty(r.prototype,"required",{get:function(){return this.requiredValue},set:function(e){this.requiredValue=u.coerceBooleanProperty(e)},enumerable:!0,configurable:!0}),r.prototype.registerOnChange=function(e){this.propagateChange=e},r.prototype.registerOnTouched=function(e){},r.prototype.ngOnInit=function(){var e=this;this.filteredMessageTypes=this.messageTypeConfigForm.get("messageType").valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(t){return e.fetchMessageTypes(t)})),f.share())},r.prototype.ngAfterViewInit=function(){},r.prototype.setDisabledState=function(e){this.disabled=e,this.disabled?this.messageTypeConfigForm.disable({emitEvent:!1}):this.messageTypeConfigForm.enable({emitEvent:!1})},r.prototype.writeValue=function(e){var t=this;this.searchText="",this.messageTypes.length=0,e&&e.forEach((function(e){var r=t.messageTypesList.find((function(t){return t.value===e}));r?t.messageTypes.push({name:r.name,value:r.value}):t.messageTypes.push({name:e,value:e})}))},r.prototype.displayMessageTypeFn=function(e){return e?e.name:void 0},r.prototype.textIsNotEmpty=function(e){return!!(e&&null!=e&&e.length>0)},r.prototype.createMessageType=function(e,t){e.preventDefault(),this.transformMessageType(t)},r.prototype.add=function(e){this.transformMessageType(e.value)},r.prototype.fetchMessageTypes=function(e){if(this.searchText=e,this.searchText&&this.searchText.length){var t=this.searchText.toUpperCase();return c.of(this.messageTypesList.filter((function(e){return e.name.toUpperCase().includes(t)})))}return c.of(this.messageTypesList)},r.prototype.transformMessageType=function(e){if((e||"").trim()){var t=null,r=e.trim(),n=this.messageTypesList.find((function(e){return e.name===r}));(t=n?{name:n.name,value:n.value}:{name:r,value:r})&&this.addMessageType(t)}this.clear("")},r.prototype.remove=function(e){var t=this.messageTypes.indexOf(e);t>=0&&(this.messageTypes.splice(t,1),this.updateModel())},r.prototype.selected=function(e){this.addMessageType(e.option.value),this.clear("")},r.prototype.addMessageType=function(e){-1===this.messageTypes.findIndex((function(t){return t.value===e.value}))&&(this.messageTypes.push(e),this.updateModel())},r.prototype.onFocus=function(){this.messageTypeConfigForm.get("messageType").updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.messageTypeInput.nativeElement.value=e,this.messageTypeConfigForm.get("messageType").patchValue(null,{emitEvent:!0}),setTimeout((function(){t.messageTypeInput.nativeElement.blur(),t.messageTypeInput.nativeElement.focus()}),0)},r.prototype.updateModel=function(){var e=this.messageTypes.map((function(e){return e.value}));this.required?(this.chipList.errorState=!e.length,this.propagateChange(e.length>0?e:null)):(this.chipList.errorState=!1,this.propagateChange(e))},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:a.TruncatePipe},{type:i.FormBuilder}]},b([t.Input(),h("design:type",Boolean),h("design:paramtypes",[Boolean])],r.prototype,"required",null),b([t.Input(),h("design:type",String)],r.prototype,"label",void 0),b([t.Input(),h("design:type",Object)],r.prototype,"placeholder",void 0),b([t.Input(),h("design:type",Boolean)],r.prototype,"disabled",void 0),b([t.ViewChild("chipList",{static:!1}),h("design:type",d.MatChipList)],r.prototype,"chipList",void 0),b([t.ViewChild("messageTypeAutocomplete",{static:!1}),h("design:type",p.MatAutocomplete)],r.prototype,"matAutocomplete",void 0),b([t.ViewChild("messageTypeInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"messageTypeInput",void 0),r=l=b([t.Component({selector:"tb-message-types-config",template:'\n {{ label }}\n \n \n {{messageType.name}}\n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-message-types-found\n
\n \n \n {{ translate.get(\'tb.rulenode.no-message-type-matching\',\n {messageType: truncate.transform(searchText, true, 6, '...')}) | async }}\n \n \n \n tb.rulenode.create-new-message-type\n \n
\n \n \n \n {{ \'tb.rulenode.message-types-required\' | translate }}\n \n\n',providers:[{provide:i.NG_VALUE_ACCESSOR,useExisting:t.forwardRef((function(){return l})),multi:!0}]}),h("design:paramtypes",[o.Store,n.TranslateService,a.TruncatePipe,i.FormBuilder])],r)}(a.PageComponent),oe=function(){function e(){}return e=b([t.NgModule({declarations:[te,re,ne,ae],imports:[r.CommonModule,a.SharedModule,m.HomeComponentsModule],exports:[te,re,ne,ae]})],e)}(),ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.unassignCustomerConfigForm},r.prototype.onConfigurationSet=function(e){this.unassignCustomerConfigForm=this.fb.group({customerNamePattern:[e?e.customerNamePattern:null,[i.Validators.required]],customerCacheExpiration:[e?e.customerCacheExpiration:null,[i.Validators.required,i.Validators.min(0)]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-un-assign-to-customer-config",template:'
\n \n tb.rulenode.customer-name-pattern\n \n \n {{ \'tb.rulenode.customer-name-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.customer-cache-expiration\n \n \n {{ \'tb.rulenode.customer-cache-expiration-required\' | translate }}\n \n \n {{ \'tb.rulenode.customer-cache-expiration-range\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.snsConfigForm},r.prototype.onConfigurationSet=function(e){this.snsConfigForm=this.fb.group({topicArnPattern:[e?e.topicArnPattern:null,[i.Validators.required]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sns-config",template:'
\n \n tb.rulenode.topic-arn-pattern\n \n \n {{ \'tb.rulenode.topic-arn-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),se=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.sqsQueueType=G,n.sqsQueueTypes=Object.keys(G),n.sqsQueueTypeTranslationsMap=z,n}return y(r,e),r.prototype.configForm=function(){return this.sqsConfigForm},r.prototype.onConfigurationSet=function(e){this.sqsConfigForm=this.fb.group({queueType:[e?e.queueType:null,[i.Validators.required]],queueUrlPattern:[e?e.queueUrlPattern:null,[i.Validators.required]],delaySeconds:[e?e.delaySeconds:null,[i.Validators.min(0),i.Validators.max(900)]],messageAttributes:[e?e.messageAttributes:null,[]],accessKeyId:[e?e.accessKeyId:null,[i.Validators.required]],secretAccessKey:[e?e.secretAccessKey:null,[i.Validators.required]],region:[e?e.region:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-sqs-config",template:'
\n \n tb.rulenode.queue-type\n \n \n {{ sqsQueueTypeTranslationsMap.get(type) | translate }}\n \n \n \n \n tb.rulenode.queue-url-pattern\n \n \n {{ \'tb.rulenode.queue-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.delay-seconds\n \n \n {{ \'tb.rulenode.min-delay-seconds-message\' | translate }}\n \n \n {{ \'tb.rulenode.max-delay-seconds-message\' | translate }}\n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n \n tb.rulenode.aws-access-key-id\n \n \n {{ \'tb.rulenode.aws-access-key-id-required\' | translate }}\n \n \n \n tb.rulenode.aws-secret-access-key\n \n \n {{ \'tb.rulenode.aws-secret-access-key-required\' | translate }}\n \n \n \n tb.rulenode.aws-region\n \n \n {{ \'tb.rulenode.aws-region-required\' | translate }}\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),me=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.pubSubConfigForm},r.prototype.onConfigurationSet=function(e){this.pubSubConfigForm=this.fb.group({projectId:[e?e.projectId:null,[i.Validators.required]],topicName:[e?e.topicName:null,[i.Validators.required]],serviceAccountKey:[e?e.serviceAccountKey:null,[i.Validators.required]],serviceAccountKeyFileName:[e?e.serviceAccountKeyFileName:null,[i.Validators.required]],messageAttributes:[e?e.messageAttributes:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-pub-sub-config",template:'
\n \n tb.rulenode.gcp-project-id\n \n \n {{ \'tb.rulenode.gcp-project-id-required\' | translate }}\n \n \n \n tb.rulenode.pubsub-topic-name\n \n \n {{ \'tb.rulenode.pubsub-topic-name-required\' | translate }}\n \n \n \n \n \n
tb.rulenode.message-attributes-hint
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ue=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.ackValues=["all","-1","0","1"],n.ToByteStandartCharsetTypesValues=W,n.ToByteStandartCharsetTypeTranslationMap=J,n}return y(r,e),r.prototype.configForm=function(){return this.kafkaConfigForm},r.prototype.onConfigurationSet=function(e){this.kafkaConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],bootstrapServers:[e?e.bootstrapServers:null,[i.Validators.required]],retries:[e?e.retries:null,[i.Validators.min(0)]],batchSize:[e?e.batchSize:null,[i.Validators.min(0)]],linger:[e?e.linger:null,[i.Validators.min(0)]],bufferMemory:[e?e.bufferMemory:null,[i.Validators.min(0)]],acks:[e?e.acks:null,[i.Validators.required]],keySerializer:[e?e.keySerializer:null,[i.Validators.required]],valueSerializer:[e?e.valueSerializer:null,[i.Validators.required]],otherProperties:[e?e.otherProperties:null,[]],addMetadataKeyValuesAsKafkaHeaders:[!!e&&e.addMetadataKeyValuesAsKafkaHeaders,[]],kafkaHeadersCharset:[e?e.kafkaHeadersCharset:null,[]]})},r.prototype.validatorTriggers=function(){return["addMetadataKeyValuesAsKafkaHeaders"]},r.prototype.updateValidators=function(e){this.kafkaConfigForm.get("addMetadataKeyValuesAsKafkaHeaders").value?this.kafkaConfigForm.get("kafkaHeadersCharset").setValidators([i.Validators.required]):this.kafkaConfigForm.get("kafkaHeadersCharset").setValidators([]),this.kafkaConfigForm.get("kafkaHeadersCharset").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-kafka-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n tb.rulenode.bootstrap-servers\n \n \n {{ \'tb.rulenode.bootstrap-servers-required\' | translate }}\n \n \n \n tb.rulenode.retries\n \n \n {{ \'tb.rulenode.min-retries-message\' | translate }}\n \n \n \n tb.rulenode.batch-size-bytes\n \n \n {{ \'tb.rulenode.min-batch-size-bytes-message\' | translate }}\n \n \n \n tb.rulenode.linger-ms\n \n \n {{ \'tb.rulenode.min-linger-ms-message\' | translate }}\n \n \n \n tb.rulenode.buffer-memory-bytes\n \n \n {{ \'tb.rulenode.min-buffer-memory-bytes-message\' | translate }}\n \n \n \n tb.rulenode.acks\n \n \n {{ ackValue }}\n \n \n \n \n tb.rulenode.key-serializer\n \n \n {{ \'tb.rulenode.key-serializer-required\' | translate }}\n \n \n \n tb.rulenode.value-serializer\n \n \n {{ \'tb.rulenode.value-serializer-required\' | translate }}\n \n \n \n \n \n \n {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}\n \n
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
\n \n tb.rulenode.charset-encoding\n \n \n {{ ToByteStandartCharsetTypeTranslationMap.get(charset) | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),de=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allMqttCredentialsTypes=$,n.mqttCredentialsTypeTranslationsMap=_,n}return y(r,e),r.prototype.configForm=function(){return this.mqttConfigForm},r.prototype.onConfigurationSet=function(e){this.mqttConfigForm=this.fb.group({topicPattern:[e?e.topicPattern:null,[i.Validators.required]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],connectTimeoutSec:[e?e.connectTimeoutSec:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(200)]],clientId:[e?e.clientId:null,[]],cleanSession:[!!e&&e.cleanSession,[]],ssl:[!!e&&e.ssl,[]],credentials:this.fb.group({type:[e&&e.credentials?e.credentials.type:null,[i.Validators.required]],username:[e&&e.credentials?e.credentials.username:null,[]],password:[e&&e.credentials?e.credentials.password:null,[]],caCert:[e&&e.credentials?e.credentials.caCert:null,[]],caCertFileName:[e&&e.credentials?e.credentials.caCertFileName:null,[]],privateKey:[e&&e.credentials?e.credentials.privateKey:null,[]],privateKeyFileName:[e&&e.credentials?e.credentials.privateKeyFileName:null,[]],cert:[e&&e.credentials?e.credentials.cert:null,[]],certFileName:[e&&e.credentials?e.credentials.certFileName:null,[]]})})},r.prototype.prepareOutputConfig=function(e){var t=e.credentials.type;switch(t){case"anonymous":e.credentials={type:t};break;case"basic":e.credentials={type:t,username:e.credentials.username,password:e.credentials.password};break;case"cert.PEM":delete e.credentials.username}return e},r.prototype.validatorTriggers=function(){return["credentials.type"]},r.prototype.updateValidators=function(e){var t=this.mqttConfigForm.get("credentials"),r=t.get("type").value;switch(e&&t.reset({type:r},{emitEvent:!1}),t.get("username").setValidators([]),t.get("password").setValidators([]),t.get("caCert").setValidators([]),t.get("caCertFileName").setValidators([]),t.get("privateKey").setValidators([]),t.get("privateKeyFileName").setValidators([]),t.get("cert").setValidators([]),t.get("certFileName").setValidators([]),r){case"anonymous":break;case"basic":t.get("username").setValidators([i.Validators.required]),t.get("password").setValidators([i.Validators.required]);break;case"cert.PEM":t.get("caCert").setValidators([i.Validators.required]),t.get("caCertFileName").setValidators([i.Validators.required]),t.get("privateKey").setValidators([i.Validators.required]),t.get("privateKeyFileName").setValidators([i.Validators.required]),t.get("cert").setValidators([i.Validators.required]),t.get("certFileName").setValidators([i.Validators.required])}t.get("username").updateValueAndValidity({emitEvent:e}),t.get("password").updateValueAndValidity({emitEvent:e}),t.get("caCert").updateValueAndValidity({emitEvent:e}),t.get("caCertFileName").updateValueAndValidity({emitEvent:e}),t.get("privateKey").updateValueAndValidity({emitEvent:e}),t.get("privateKeyFileName").updateValueAndValidity({emitEvent:e}),t.get("cert").updateValueAndValidity({emitEvent:e}),t.get("certFileName").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-mqtt-config",template:'
\n \n tb.rulenode.topic-pattern\n \n \n {{ \'tb.rulenode.topic-pattern-required\' | translate }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n \n tb.rulenode.connect-timeout\n \n \n {{ \'tb.rulenode.connect-timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n {{ \'tb.rulenode.connect-timeout-range\' | translate }}\n \n \n
\n \n tb.rulenode.client-id\n \n \n \n {{ \'tb.rulenode.clean-session\' | translate }}\n \n \n {{ \'tb.rulenode.enable-ssl\' | translate }}\n \n \n \n tb.rulenode.credentials\n \n {{ mqttCredentialsTypeTranslationsMap.get(mqttConfigForm.get(\'credentials\').get(\'type\').value) | translate }}\n \n \n
\n \n tb.rulenode.credentials-type\n \n \n {{ mqttCredentialsTypeTranslationsMap.get(credentialsType) | translate }}\n \n \n \n {{ \'tb.rulenode.credentials-type-required\' | translate }}\n \n \n
\n \n \n \n \n tb.rulenode.username\n \n \n {{ \'tb.rulenode.username-required\' | translate }}\n \n \n \n tb.rulenode.password\n \n \n {{ \'tb.rulenode.password-required\' | translate }}\n \n \n \n \n \n \n \n \n \n \n \n tb.rulenode.private-key-password\n \n \n \n
\n
\n \n
\n',styles:[":host .tb-mqtt-credentials-panel-group{margin:0 6px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),pe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],n}return y(r,e),r.prototype.configForm=function(){return this.rabbitMqConfigForm},r.prototype.onConfigurationSet=function(e){this.rabbitMqConfigForm=this.fb.group({exchangeNamePattern:[e?e.exchangeNamePattern:null,[]],routingKeyPattern:[e?e.routingKeyPattern:null,[]],messageProperties:[e?e.messageProperties:null,[]],host:[e?e.host:null,[i.Validators.required]],port:[e?e.port:null,[i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]],virtualHost:[e?e.virtualHost:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]],automaticRecoveryEnabled:[!!e&&e.automaticRecoveryEnabled,[]],connectionTimeout:[e?e.connectionTimeout:null,[i.Validators.min(0)]],handshakeTimeout:[e?e.handshakeTimeout:null,[i.Validators.min(0)]],clientProperties:[e?e.clientProperties:null,[]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rabbit-mq-config",template:'
\n \n tb.rulenode.exchange-name-pattern\n \n \n \n tb.rulenode.routing-key-pattern\n \n \n \n tb.rulenode.message-properties\n \n \n {{ property }}\n \n \n \n
\n \n tb.rulenode.host\n \n \n {{ \'tb.rulenode.host-required\' | translate }}\n \n \n \n tb.rulenode.port\n \n \n {{ \'tb.rulenode.port-required\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n {{ \'tb.rulenode.port-range\' | translate }}\n \n \n
\n \n tb.rulenode.virtual-host\n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n \n {{ \'tb.rulenode.automatic-recovery\' | translate }}\n \n \n tb.rulenode.connection-timeout-ms\n \n \n {{ \'tb.rulenode.min-connection-timeout-ms-message\' | translate }}\n \n \n \n tb.rulenode.handshake-timeout-ms\n \n \n {{ \'tb.rulenode.min-handshake-timeout-ms-message\' | translate }}\n \n \n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ce=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.httpRequestTypes=Object.keys(Q),n}return y(r,e),r.prototype.configForm=function(){return this.restApiCallConfigForm},r.prototype.onConfigurationSet=function(e){this.restApiCallConfigForm=this.fb.group({restEndpointUrlPattern:[e?e.restEndpointUrlPattern:null,[i.Validators.required]],requestMethod:[e?e.requestMethod:null,[i.Validators.required]],useSimpleClientHttpFactory:[!!e&&e.useSimpleClientHttpFactory,[]],readTimeoutMs:[e?e.readTimeoutMs:null,[]],maxParallelRequestsCount:[e?e.maxParallelRequestsCount:null,[i.Validators.min(0)]],headers:[e?e.headers:null,[]],useRedisQueueForMsgPersistence:[!!e&&e.useRedisQueueForMsgPersistence,[]],trimQueue:[!!e&&e.trimQueue,[]],maxQueueSize:[e?e.maxQueueSize:null,[]]})},r.prototype.validatorTriggers=function(){return["useSimpleClientHttpFactory","useRedisQueueForMsgPersistence"]},r.prototype.updateValidators=function(e){var t=this.restApiCallConfigForm.get("useSimpleClientHttpFactory").value,r=this.restApiCallConfigForm.get("useRedisQueueForMsgPersistence").value;t?this.restApiCallConfigForm.get("readTimeoutMs").setValidators([]):this.restApiCallConfigForm.get("readTimeoutMs").setValidators([i.Validators.min(0)]),r?this.restApiCallConfigForm.get("maxQueueSize").setValidators([i.Validators.min(0)]):this.restApiCallConfigForm.get("maxQueueSize").setValidators([]),this.restApiCallConfigForm.get("readTimeoutMs").updateValueAndValidity({emitEvent:e}),this.restApiCallConfigForm.get("maxQueueSize").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-rest-api-call-config",template:'
\n \n tb.rulenode.endpoint-url-pattern\n \n \n {{ \'tb.rulenode.endpoint-url-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.request-method\n \n \n {{ requestType }}\n \n \n \n \n {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}\n \n \n tb.rulenode.read-timeout\n \n \n \n \n tb.rulenode.max-parallel-requests-count\n \n \n \n \n
tb.rulenode.headers-hint
\n \n \n \n {{ \'tb.rulenode.use-redis-queue\' | translate }}\n \n
\n \n {{ \'tb.rulenode.trim-redis-queue\' | translate }}\n \n \n tb.rulenode.redis-queue-max-size\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),fe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.smtpProtocols=["smtp","smtps"],n.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],n}return y(r,e),r.prototype.configForm=function(){return this.sendEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.sendEmailConfigForm=this.fb.group({useSystemSmtpSettings:[!!e&&e.useSystemSmtpSettings,[]],smtpProtocol:[e?e.smtpProtocol:null,[]],smtpHost:[e?e.smtpHost:null,[]],smtpPort:[e?e.smtpPort:null,[]],timeout:[e?e.timeout:null,[]],enableTls:[!!e&&e.enableTls,[]],tlsVersion:[e?e.tlsVersion:null,[]],username:[e?e.username:null,[]],password:[e?e.password:null,[]]})},r.prototype.validatorTriggers=function(){return["useSystemSmtpSettings"]},r.prototype.updateValidators=function(e){this.sendEmailConfigForm.get("useSystemSmtpSettings").value?(this.sendEmailConfigForm.get("smtpProtocol").setValidators([]),this.sendEmailConfigForm.get("smtpHost").setValidators([]),this.sendEmailConfigForm.get("smtpPort").setValidators([]),this.sendEmailConfigForm.get("timeout").setValidators([])):(this.sendEmailConfigForm.get("smtpProtocol").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpHost").setValidators([i.Validators.required]),this.sendEmailConfigForm.get("smtpPort").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(65535)]),this.sendEmailConfigForm.get("timeout").setValidators([i.Validators.required,i.Validators.min(0)])),this.sendEmailConfigForm.get("smtpProtocol").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpHost").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("smtpPort").updateValueAndValidity({emitEvent:e}),this.sendEmailConfigForm.get("timeout").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-send-email-config",template:'
\n \n {{ \'tb.rulenode.use-system-smtp-settings\' | translate }}\n \n
\n \n tb.rulenode.smtp-protocol\n \n \n {{ smtpProtocol.toUpperCase() }}\n \n \n \n
\n \n tb.rulenode.smtp-host\n \n \n {{ \'tb.rulenode.smtp-host-required\' | translate }}\n \n \n \n tb.rulenode.smtp-port\n \n \n {{ \'tb.rulenode.smtp-port-required\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n {{ \'tb.rulenode.smtp-port-range\' | translate }}\n \n \n
\n \n tb.rulenode.timeout-msec\n \n \n {{ \'tb.rulenode.timeout-required\' | translate }}\n \n \n {{ \'tb.rulenode.min-timeout-msec-message\' | translate }}\n \n \n \n {{ \'tb.rulenode.enable-tls\' | translate }}\n \n \n tb.rulenode.tls-version\n \n \n {{ tlsVersion }}\n \n \n \n \n tb.rulenode.username\n \n \n \n tb.rulenode.password\n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ge=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.serviceType=a.ServiceType.TB_RULE_ENGINE,n}return y(r,e),r.prototype.configForm=function(){return this.checkPointConfigForm},r.prototype.onConfigurationSet=function(e){this.checkPointConfigForm=this.fb.group({queueName:[e?e.queueName:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-action-node-check-point-config",template:'
\n \n \n
tb.rulenode.select-queue-hint
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ye=function(){function e(){}return e=b([t.NgModule({declarations:[T,x,q,S,I,k,N,V,E,A,L,Y,Z,X,ee,ie,le,se,me,ue,de,pe,ce,fe,ge],imports:[r.CommonModule,a.SharedModule,oe],exports:[T,x,q,S,I,k,N,V,E,A,L,Y,Z,X,ee,ie,le,se,me,ue,de,pe,ce,fe,ge]})],e)}(),be=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.checkMessageConfigForm},r.prototype.onConfigurationSet=function(e){this.checkMessageConfigForm=this.fb.group({messageNames:[e?e.messageNames:null,[]],metadataNames:[e?e.metadataNames:null,[]],checkAllKeys:[!!e&&e.checkAllKeys,[]]})},r.prototype.validateConfig=function(){var e=this.checkMessageConfigForm.get("messageNames").value,t=this.checkMessageConfigForm.get("metadataNames").value;return e.length>0||t.length>0},r.prototype.removeMessageName=function(e){var t=this.checkMessageConfigForm.get("messageNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("messageNames").setValue(t,{emitEvent:!0}))},r.prototype.removeMetadataName=function(e){var t=this.checkMessageConfigForm.get("metadataNames").value,r=t.indexOf(e);r>=0&&(t.splice(r,1),this.checkMessageConfigForm.get("metadataNames").setValue(t,{emitEvent:!0}))},r.prototype.addMessageName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("messageNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("messageNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.prototype.addMetadataName=function(e){var t=e.input,r=e.value;if((r||"").trim()){r=r.trim();var n=this.checkMessageConfigForm.get("metadataNames").value;n&&-1!==n.indexOf(r)||(n||(n=[]),n.push(r),this.checkMessageConfigForm.get("metadataNames").setValue(n,{emitEvent:!0}))}t&&(t.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-message-config",template:'
\n \n \n \n \n \n {{messageName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n \n \n \n \n {{metadataName}}\n close\n \n \n \n \n
tb.rulenode.separator-hint
\n \n {{ \'tb.rulenode.check-all-keys\' | translate }}\n \n
tb.rulenode.check-all-keys-hint
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),he=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.entitySearchDirection=Object.keys(a.EntitySearchDirection),n.entitySearchDirectionTranslationsMap=a.entitySearchDirectionTranslations,n}return y(r,e),r.prototype.configForm=function(){return this.checkRelationConfigForm},r.prototype.onConfigurationSet=function(e){this.checkRelationConfigForm=this.fb.group({checkForSingleEntity:[!!e&&e.checkForSingleEntity,[]],direction:[e?e.direction:null,[]],entityType:[e?e.entityType:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],entityId:[e?e.entityId:null,e&&e.checkForSingleEntity?[i.Validators.required]:[]],relationType:[e?e.relationType:null,[i.Validators.required]]})},r.prototype.validatorTriggers=function(){return["checkForSingleEntity"]},r.prototype.updateValidators=function(e){var t=this.checkRelationConfigForm.get("checkForSingleEntity").value;this.checkRelationConfigForm.get("entityType").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityType").updateValueAndValidity({emitEvent:e}),this.checkRelationConfigForm.get("entityId").setValidators(t?[i.Validators.required]:[]),this.checkRelationConfigForm.get("entityId").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-check-relation-config",template:'
\n \n {{ \'tb.rulenode.check-relation-to-specific-entity\' | translate }}\n \n
tb.rulenode.check-relation-hint
\n \n relation.direction\n \n \n {{ entitySearchDirectionTranslationsMap.get(direction) | translate }}\n \n \n \n
\n \n \n \n \n
\n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ce=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.perimeterType=M,n.perimeterTypes=Object.keys(M),n.perimeterTypeTranslationMap=w,n.rangeUnits=Object.keys(O),n.rangeUnitTranslationMap=B,n}return y(r,e),r.prototype.configForm=function(){return this.geoFilterConfigForm},r.prototype.onConfigurationSet=function(e){this.geoFilterConfigForm=this.fb.group({latitudeKeyName:[e?e.latitudeKeyName:null,[i.Validators.required]],longitudeKeyName:[e?e.longitudeKeyName:null,[i.Validators.required]],fetchPerimeterInfoFromMessageMetadata:[!!e&&e.fetchPerimeterInfoFromMessageMetadata,[]],perimeterType:[e?e.perimeterType:null,[]],centerLatitude:[e?e.centerLatitude:null,[]],centerLongitude:[e?e.centerLatitude:null,[]],range:[e?e.range:null,[]],rangeUnit:[e?e.rangeUnit:null,[]],polygonsDefinition:[e?e.polygonsDefinition:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchPerimeterInfoFromMessageMetadata","perimeterType"]},r.prototype.updateValidators=function(e){var t=this.geoFilterConfigForm.get("fetchPerimeterInfoFromMessageMetadata").value,r=this.geoFilterConfigForm.get("perimeterType").value;t?this.geoFilterConfigForm.get("perimeterType").setValidators([]):this.geoFilterConfigForm.get("perimeterType").setValidators([i.Validators.required]),t||r!==M.CIRCLE?(this.geoFilterConfigForm.get("centerLatitude").setValidators([]),this.geoFilterConfigForm.get("centerLongitude").setValidators([]),this.geoFilterConfigForm.get("range").setValidators([]),this.geoFilterConfigForm.get("rangeUnit").setValidators([])):(this.geoFilterConfigForm.get("centerLatitude").setValidators([i.Validators.required,i.Validators.min(-90),i.Validators.max(90)]),this.geoFilterConfigForm.get("centerLongitude").setValidators([i.Validators.required,i.Validators.min(-180),i.Validators.max(180)]),this.geoFilterConfigForm.get("range").setValidators([i.Validators.required,i.Validators.min(0)]),this.geoFilterConfigForm.get("rangeUnit").setValidators([i.Validators.required])),t||r!==M.POLYGON?this.geoFilterConfigForm.get("polygonsDefinition").setValidators([]):this.geoFilterConfigForm.get("polygonsDefinition").setValidators([i.Validators.required]),this.geoFilterConfigForm.get("perimeterType").updateValueAndValidity({emitEvent:!1}),this.geoFilterConfigForm.get("centerLatitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("centerLongitude").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("range").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("rangeUnit").updateValueAndValidity({emitEvent:e}),this.geoFilterConfigForm.get("polygonsDefinition").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-gps-geofencing-config",template:'
\n \n tb.rulenode.latitude-key-name\n \n \n {{ \'tb.rulenode.latitude-key-name-required\' | translate }}\n \n \n \n tb.rulenode.longitude-key-name\n \n \n {{ \'tb.rulenode.longitude-key-name-required\' | translate }}\n \n \n \n {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}\n \n
\n \n tb.rulenode.perimeter-type\n \n \n {{ perimeterTypeTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n \n tb.rulenode.circle-center-latitude\n \n \n {{ \'tb.rulenode.circle-center-latitude-required\' | translate }}\n \n \n \n tb.rulenode.circle-center-longitude\n \n \n {{ \'tb.rulenode.circle-center-longitude-required\' | translate }}\n \n \n
\n
\n \n tb.rulenode.range\n \n \n {{ \'tb.rulenode.range-required\' | translate }}\n \n \n \n tb.rulenode.range-units\n \n \n {{ rangeUnitTranslationMap.get(type) | translate }}\n \n \n \n
\n
\n
\n
\n \n tb.rulenode.polygon-definition\n \n \n {{ \'tb.rulenode.polygon-definition-required\' | translate }}\n \n \n
\n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ve=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.messageTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.messageTypeConfigForm=this.fb.group({messageTypes:[e?e.messageTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-message-type-config",template:'
\n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Fe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.allowedEntityTypes=[a.EntityType.DEVICE,a.EntityType.ASSET,a.EntityType.ENTITY_VIEW,a.EntityType.TENANT,a.EntityType.CUSTOMER,a.EntityType.USER,a.EntityType.DASHBOARD,a.EntityType.RULE_CHAIN,a.EntityType.RULE_NODE],n}return y(r,e),r.prototype.configForm=function(){return this.originatorTypeConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorTypeConfigForm=this.fb.group({originatorTypes:[e?e.originatorTypes:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-filter-node-originator-type-config",template:'
\n \n \n \n
\n',styles:[":host ::ng-deep tb-entity-type-list .mat-form-field-flex{padding-top:0}:host ::ng-deep tb-entity-type-list .mat-form-field-infix{border-top:0}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Te=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"filter",this.translate.instant("tb.rulenode.filter"),"Filter",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),xe=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.switchConfigForm},r.prototype.onConfigurationSet=function(e){this.switchConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.switchConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"switch",this.translate.instant("tb.rulenode.switch"),"Switch",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.switchConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-filter-node-switch-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),qe=function(e){function r(t,r,n){var o,l,s=e.call(this,t)||this;s.store=t,s.translate=r,s.fb=n,s.alarmStatusTranslationsMap=a.alarmStatusTranslations,s.alarmStatusList=[],s.searchText="",s.displayStatusFn=s.displayStatus.bind(s);try{for(var m=C(Object.keys(a.AlarmStatus)),u=m.next();!u.done;u=m.next()){var d=u.value;s.alarmStatusList.push(a.AlarmStatus[d])}}catch(e){o={error:e}}finally{try{u&&!u.done&&(l=m.return)&&l.call(m)}finally{if(o)throw o.error}}return s.statusFormControl=new i.FormControl(""),s.filteredAlarmStatus=s.statusFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return s.fetchAlarmStatus(e)})),f.share()),s}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.alarmStatusConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.statusFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.alarmStatusConfigForm=this.fb.group({alarmStatusList:[e?e.alarmStatusList:null,[i.Validators.required]]})},r.prototype.displayStatus=function(e){return e?this.translate.instant(a.alarmStatusTranslations.get(e)):void 0},r.prototype.fetchAlarmStatus=function(e){var t=this,r=this.getAlarmStatusList();if(this.searchText=e,this.searchText&&this.searchText.length){var n=this.searchText.toUpperCase();return c.of(r.filter((function(e){return t.translate.instant(a.alarmStatusTranslations.get(a.AlarmStatus[e])).toUpperCase().includes(n)})))}return c.of(r)},r.prototype.alarmStatusSelected=function(e){this.addAlarmStatus(e.option.value),this.clear("")},r.prototype.removeAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))}},r.prototype.addAlarmStatus=function(e){var t=this.alarmStatusConfigForm.get("alarmStatusList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.alarmStatusConfigForm.get("alarmStatusList").setValue(t))},r.prototype.getAlarmStatusList=function(){var e=this;return this.alarmStatusList.filter((function(t){return-1===e.alarmStatusConfigForm.get("alarmStatusList").value.indexOf(t)}))},r.prototype.onAlarmStatusInputFocus=function(){this.statusFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.alarmStatusInput.nativeElement.value=e,this.statusFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.alarmStatusInput.nativeElement.blur(),t.alarmStatusInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("alarmStatusInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"alarmStatusInput",void 0),r=b([t.Component({selector:"tb-filter-node-check-alarm-status-config",template:'
\n \n tb.rulenode.alarm-status-filter\n \n \n \n {{alarmStatusTranslationsMap.get(alarmStatus) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-alarm-status-matching\n
\n
\n \n \n \n \n
\n\n\n\n'}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Se=function(){function e(){}return e=b([t.NgModule({declarations:[be,he,Ce,ve,Fe,Te,xe,qe],imports:[r.CommonModule,a.SharedModule,oe],exports:[be,he,Ce,ve,Fe,Te,xe,qe]})],e)}(),Ie=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.customerAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.customerAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-customer-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),ke=function(e){function r(t,r,n){var a,o,l=e.call(this,t)||this;l.store=t,l.translate=r,l.fb=n,l.entityDetailsTranslationsMap=H,l.entityDetailsList=[],l.searchText="",l.displayDetailsFn=l.displayDetails.bind(l);try{for(var s=C(Object.keys(K)),m=s.next();!m.done;m=s.next()){var u=m.value;l.entityDetailsList.push(K[u])}}catch(e){a={error:e}}finally{try{m&&!m.done&&(o=s.return)&&o.call(s)}finally{if(a)throw a.error}}return l.detailsFormControl=new i.FormControl(""),l.filteredEntityDetails=l.detailsFormControl.valueChanges.pipe(f.startWith(""),f.map((function(e){return e||""})),f.mergeMap((function(e){return l.fetchEntityDetails(e)})),f.share()),l}return y(r,e),r.prototype.ngOnInit=function(){e.prototype.ngOnInit.call(this)},r.prototype.configForm=function(){return this.entityDetailsConfigForm},r.prototype.prepareInputConfig=function(e){return this.searchText="",this.detailsFormControl.patchValue("",{emitEvent:!0}),e},r.prototype.onConfigurationSet=function(e){this.entityDetailsConfigForm=this.fb.group({detailsList:[e?e.detailsList:null,[i.Validators.required]],addToMetadata:[!!e&&e.addToMetadata,[]]})},r.prototype.displayDetails=function(e){return e?this.translate.instant(H.get(e)):void 0},r.prototype.fetchEntityDetails=function(e){var t=this;if(this.searchText=e,this.searchText&&this.searchText.length){var r=this.searchText.toUpperCase();return c.of(this.entityDetailsList.filter((function(e){return t.translate.instant(H.get(K[e])).toUpperCase().includes(r)})))}return c.of(this.entityDetailsList)},r.prototype.detailsFieldSelected=function(e){this.addDetailsField(e.option.value),this.clear("")},r.prototype.removeDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;if(t){var r=t.indexOf(e);r>=0&&(t.splice(r,1),this.entityDetailsConfigForm.get("detailsList").setValue(t))}},r.prototype.addDetailsField=function(e){var t=this.entityDetailsConfigForm.get("detailsList").value;t||(t=[]),-1===t.indexOf(e)&&(t.push(e),this.entityDetailsConfigForm.get("detailsList").setValue(t))},r.prototype.onEntityDetailsInputFocus=function(){this.detailsFormControl.updateValueAndValidity({onlySelf:!0,emitEvent:!0})},r.prototype.clear=function(e){var t=this;void 0===e&&(e=""),this.detailsInput.nativeElement.value=e,this.detailsFormControl.patchValue(null,{emitEvent:!0}),setTimeout((function(){t.detailsInput.nativeElement.blur(),t.detailsInput.nativeElement.focus()}),0)},r.ctorParameters=function(){return[{type:o.Store},{type:n.TranslateService},{type:i.FormBuilder}]},b([t.ViewChild("detailsInput",{static:!1}),h("design:type",t.ElementRef)],r.prototype,"detailsInput",void 0),r=b([t.Component({selector:"tb-enrichment-node-entity-details-config",template:'
\n \n tb.rulenode.entity-details\n \n \n \n {{entityDetailsTranslationsMap.get(details) | translate}}\n \n close\n \n \n \n \n \n \n \n \n
\n
\n tb.rulenode.no-entity-details-matching\n
\n
\n \n \n \n \n \n {{ \'tb.rulenode.add-to-metadata\' | translate }}\n \n
tb.rulenode.add-to-metadata-hint
\n
\n',styles:[":host ::ng-deep mat-form-field.entity-fields-list .mat-form-field-wrapper{margin-bottom:-1.25em}"]}),h("design:paramtypes",[o.Store,n.TranslateService,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ne=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.deviceAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.deviceAttributesConfigForm=this.fb.group({deviceRelationsQuery:[e?e.deviceRelationsQuery:null,[i.Validators.required]],tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.deviceAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.deviceAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.deviceAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.deviceAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-device-attributes-config",template:'
\n \n \n \n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ve=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n}return y(r,e),r.prototype.configForm=function(){return this.originatorAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorAttributesConfigForm=this.fb.group({tellFailureIfAbsent:[!!e&&e.tellFailureIfAbsent,[]],clientAttributeNames:[e?e.clientAttributeNames:null,[]],sharedAttributeNames:[e?e.sharedAttributeNames:null,[]],serverAttributeNames:[e?e.serverAttributeNames:null,[]],latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],getLatestValueWithTs:[!!e&&e.getLatestValueWithTs,[]]})},r.prototype.removeKey=function(e,t){var r=this.originatorAttributesConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.originatorAttributesConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.originatorAttributesConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.originatorAttributesConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-attributes-config",template:'
\n \n {{ \'tb.rulenode.tell-failure-if-absent\' | translate }}\n \n
tb.rulenode.tell-failure-if-absent-hint
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n {{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}\n \n
\n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ee=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.originatorFieldsConfigForm},r.prototype.onConfigurationSet=function(e){this.originatorFieldsConfigForm=this.fb.group({fieldsMapping:[e?e.fieldsMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-originator-fields-config",template:'
\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Ae=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.separatorKeysCodes=[s.ENTER,s.COMMA,s.SEMICOLON],n.fetchMode=j,n.fetchModes=Object.keys(j),n.samplingOrders=Object.keys(U),n.timeUnits=Object.keys(R),n.timeUnitsTranslationMap=D,n}return y(r,e),r.prototype.configForm=function(){return this.getTelemetryFromDatabaseConfigForm},r.prototype.onConfigurationSet=function(e){this.getTelemetryFromDatabaseConfigForm=this.fb.group({latestTsKeyNames:[e?e.latestTsKeyNames:null,[]],fetchMode:[e?e.fetchMode:null,[i.Validators.required]],orderBy:[e?e.orderBy:null,[]],limit:[e?e.limit:null,[]],useMetadataIntervalPatterns:[!!e&&e.useMetadataIntervalPatterns,[]],startInterval:[e?e.startInterval:null,[]],startIntervalTimeUnit:[e?e.startIntervalTimeUnit:null,[]],endInterval:[e?e.endInterval:null,[]],endIntervalTimeUnit:[e?e.endIntervalTimeUnit:null,[]],startIntervalPattern:[e?e.startIntervalPattern:null,[]],endIntervalPattern:[e?e.endIntervalPattern:null,[]]})},r.prototype.validatorTriggers=function(){return["fetchMode","useMetadataIntervalPatterns"]},r.prototype.updateValidators=function(e){var t=this.getTelemetryFromDatabaseConfigForm.get("fetchMode").value,r=this.getTelemetryFromDatabaseConfigForm.get("useMetadataIntervalPatterns").value;t&&t===j.ALL?(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([i.Validators.required,i.Validators.min(2),i.Validators.max(1e3)])):(this.getTelemetryFromDatabaseConfigForm.get("orderBy").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("limit").setValidators([])),r?(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([i.Validators.required])):(this.getTelemetryFromDatabaseConfigForm.get("startInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("endInterval").setValidators([i.Validators.required,i.Validators.min(1),i.Validators.max(2147483647)]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").setValidators([i.Validators.required]),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").setValidators([]),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").setValidators([])),this.getTelemetryFromDatabaseConfigForm.get("orderBy").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("limit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endInterval").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalTimeUnit").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("startIntervalPattern").updateValueAndValidity({emitEvent:e}),this.getTelemetryFromDatabaseConfigForm.get("endIntervalPattern").updateValueAndValidity({emitEvent:e})},r.prototype.removeKey=function(e,t){var r=this.getTelemetryFromDatabaseConfigForm.get(t).value,n=r.indexOf(e);n>=0&&(r.splice(n,1),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(r,{emitEvent:!0}))},r.prototype.addKey=function(e,t){var r=e.input,n=e.value;if((n||"").trim()){n=n.trim();var a=this.getTelemetryFromDatabaseConfigForm.get(t).value;a&&-1!==a.indexOf(n)||(a||(a=[]),a.push(n),this.getTelemetryFromDatabaseConfigForm.get(t).setValue(a,{emitEvent:!0}))}r&&(r.value="")},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-get-telemetry-from-database",template:'
\n \n \n \n \n \n {{key}}\n close\n \n \n \n \n \n tb.rulenode.fetch-mode\n \n \n {{ mode }}\n \n \n tb.rulenode.fetch-mode-hint\n \n
\n \n tb.rulenode.order-by\n \n \n {{ order }}\n \n \n tb.rulenode.order-by-hint\n \n \n tb.rulenode.limit\n \n tb.rulenode.limit-hint\n \n
\n \n {{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}\n \n
tb.rulenode.use-metadata-interval-patterns-hint
\n
\n
\n \n tb.rulenode.start-interval\n \n \n {{ \'tb.rulenode.start-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.start-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n tb.rulenode.end-interval\n \n \n {{ \'tb.rulenode.end-interval-value-required\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n {{ \'tb.rulenode.time-value-range\' | translate }}\n \n \n \n tb.rulenode.end-interval-time-unit\n \n \n {{ timeUnitsTranslationMap.get(timeUnit) | translate }}\n \n \n \n
\n
\n \n \n tb.rulenode.start-interval-pattern\n \n \n {{ \'tb.rulenode.start-interval-pattern-required\' | translate }}\n \n \n \n \n tb.rulenode.end-interval-pattern\n \n \n {{ \'tb.rulenode.end-interval-pattern-required\' | translate }}\n \n \n \n \n
\n',styles:[":host label.tb-title{margin-bottom:-10px}"]}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Le=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.relatedAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.relatedAttributesConfigForm=this.fb.group({relationsQuery:[e?e.relationsQuery:null,[i.Validators.required]],telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-related-attributes-config",template:'
\n \n \n \n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Me=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.tenantAttributesConfigForm},r.prototype.onConfigurationSet=function(e){this.tenantAttributesConfigForm=this.fb.group({telemetry:[!!e&&e.telemetry,[]],attrMapping:[e?e.attrMapping:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-enrichment-node-tenant-attributes-config",template:'
\n \n \n {{ \'tb.rulenode.latest-telemetry\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),Pe=function(){function e(){}return e=b([t.NgModule({declarations:[Ie,ke,Ne,Ve,Ee,Ae,Le,Me],imports:[r.CommonModule,a.SharedModule,oe],exports:[Ie,ke,Ne,Ve,Ee,Ae,Le,Me]})],e)}(),Re=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n.originatorSource=v,n.originatorSources=Object.keys(v),n.originatorSourceTranslationMap=P,n}return y(r,e),r.prototype.configForm=function(){return this.changeOriginatorConfigForm},r.prototype.onConfigurationSet=function(e){this.changeOriginatorConfigForm=this.fb.group({originatorSource:[e?e.originatorSource:null,[i.Validators.required]],relationsQuery:[e?e.relationsQuery:null,[]]})},r.prototype.validatorTriggers=function(){return["originatorSource"]},r.prototype.updateValidators=function(e){var t=this.changeOriginatorConfigForm.get("originatorSource").value;t&&t===v.RELATED?this.changeOriginatorConfigForm.get("relationsQuery").setValidators([i.Validators.required]):this.changeOriginatorConfigForm.get("relationsQuery").setValidators([]),this.changeOriginatorConfigForm.get("relationsQuery").updateValueAndValidity({emitEvent:e})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-change-originator-config",template:'
\n \n tb.rulenode.originator-source\n \n \n {{ originatorSourceTranslationMap.get(source) | translate }}\n \n \n \n
\n \n \n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),we=function(e){function r(t,r,n,a){var o=e.call(this,t)||this;return o.store=t,o.fb=r,o.nodeScriptTestService=n,o.translate=a,o}return y(r,e),r.prototype.configForm=function(){return this.scriptConfigForm},r.prototype.onConfigurationSet=function(e){this.scriptConfigForm=this.fb.group({jsScript:[e?e.jsScript:null,[i.Validators.required]]})},r.prototype.testScript=function(){var e=this,t=this.scriptConfigForm.get("jsScript").value;this.nodeScriptTestService.testNodeScript(t,"update",this.translate.instant("tb.rulenode.transformer"),"Transform",["msg","metadata","msgType"],this.ruleNodeId).subscribe((function(t){t&&e.scriptConfigForm.get("jsScript").setValue(t)}))},r.prototype.onValidate=function(){this.jsFuncComponent.validateOnSubmit()},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder},{type:l.NodeScriptTestService},{type:n.TranslateService}]},b([t.ViewChild("jsFuncComponent",{static:!0}),h("design:type",a.JsFuncComponent)],r.prototype,"jsFuncComponent",void 0),r=b([t.Component({selector:"tb-transformation-node-script-config",template:'
\n \n \n \n
\n \n
\n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder,l.NodeScriptTestService,n.TranslateService])],r)}(a.RuleNodeConfigurationComponent),Oe=function(e){function r(t,r){var n=e.call(this,t)||this;return n.store=t,n.fb=r,n}return y(r,e),r.prototype.configForm=function(){return this.toEmailConfigForm},r.prototype.onConfigurationSet=function(e){this.toEmailConfigForm=this.fb.group({fromTemplate:[e?e.fromTemplate:null,[i.Validators.required]],toTemplate:[e?e.toTemplate:null,[i.Validators.required]],ccTemplate:[e?e.ccTemplate:null,[]],bccTemplate:[e?e.bccTemplate:null,[]],subjectTemplate:[e?e.subjectTemplate:null,[i.Validators.required]],bodyTemplate:[e?e.bodyTemplate:null,[i.Validators.required]]})},r.ctorParameters=function(){return[{type:o.Store},{type:i.FormBuilder}]},r=b([t.Component({selector:"tb-transformation-node-to-email-config",template:'
\n \n tb.rulenode.from-template\n \n \n {{ \'tb.rulenode.from-template-required\' | translate }}\n \n \n \n \n tb.rulenode.to-template\n \n \n {{ \'tb.rulenode.to-template-required\' | translate }}\n \n \n \n \n tb.rulenode.cc-template\n \n \n \n \n tb.rulenode.bcc-template\n \n \n \n \n tb.rulenode.subject-template\n \n \n {{ \'tb.rulenode.subject-template-required\' | translate }}\n \n \n \n \n tb.rulenode.body-template\n \n \n {{ \'tb.rulenode.body-template-required\' | translate }}\n \n \n \n
\n'}),h("design:paramtypes",[o.Store,i.FormBuilder])],r)}(a.RuleNodeConfigurationComponent),De=function(){function e(){}return e=b([t.NgModule({declarations:[Re,we,Oe],imports:[r.CommonModule,a.SharedModule,oe],exports:[Re,we,Oe]})],e)}(),Ke=function(){function e(e){!function(e){e.setTranslation("en_US",{tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-status-filter":"Alarm status filter","alarm-status-list-empty":"Alarm status list is empty","no-alarm-status-matching":"No alarm status matching were found.",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}},!0)}(e)}return e.ctorParameters=function(){return[{type:n.TranslateService}]},e=b([t.NgModule({declarations:[F],imports:[r.CommonModule,a.SharedModule],exports:[ye,Se,Pe,De,F]}),h("design:paramtypes",[n.TranslateService])],e)}();e.RuleNodeCoreConfigModule=Ke,e.ɵa=F,e.ɵb=ye,e.ɵba=ge,e.ɵbb=oe,e.ɵbc=te,e.ɵbd=re,e.ɵbe=ne,e.ɵbf=ae,e.ɵbg=Se,e.ɵbh=be,e.ɵbi=he,e.ɵbj=Ce,e.ɵbk=ve,e.ɵbl=Fe,e.ɵbm=Te,e.ɵbn=xe,e.ɵbo=qe,e.ɵbp=Pe,e.ɵbq=Ie,e.ɵbr=ke,e.ɵbs=Ne,e.ɵbt=Ve,e.ɵbu=Ee,e.ɵbv=Ae,e.ɵbw=Le,e.ɵbx=Me,e.ɵby=De,e.ɵbz=Re,e.ɵc=T,e.ɵca=we,e.ɵcb=Oe,e.ɵd=x,e.ɵe=q,e.ɵf=S,e.ɵg=I,e.ɵh=k,e.ɵi=N,e.ɵj=V,e.ɵk=E,e.ɵl=A,e.ɵm=L,e.ɵn=Y,e.ɵo=Z,e.ɵp=X,e.ɵq=ee,e.ɵr=ie,e.ɵs=le,e.ɵt=se,e.ɵu=me,e.ɵv=ue,e.ɵw=de,e.ɵx=pe,e.ɵy=ce,e.ɵz=fe,Object.defineProperty(e,"__esModule",{value:!0})})); -//# sourceMappingURL=rulenode-core-config.umd.min.js.map +//# sourceMappingURL=rulenode-core-config.umd.min.js.map \ No newline at end of file From 0289b0c1a2b9bec66cc5261c3c862a855d01d140 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 27 Apr 2020 10:39:18 +0300 Subject: [PATCH 100/102] UI: Get rid of 'multiple directives with 'exportAs' set to ngForm' --- .../components/alias/entity-alias-dialog.component.html | 2 +- .../alias/entity-aliases-dialog.component.html | 2 +- .../add-widget-to-dashboard-dialog.component.html | 2 +- .../attribute/edit-attribute-value-panel.component.html | 3 +-- .../dashboard/select-target-layout-dialog.component.html | 2 +- .../dashboard/select-target-state-dialog.component.html | 2 +- .../modules/home/components/details-panel.component.ts | 8 ++++---- .../components/entity/add-entity-dialog.component.ts | 6 +++--- .../components/entity/entity-details-panel.component.ts | 6 +++--- .../home/components/entity/entity-tabs.component.ts | 4 ++-- .../modules/home/components/entity/entity.component.ts | 2 -- .../import-export/import-dialog.component.html | 2 +- .../components/relation/relation-dialog.component.html | 2 +- .../widget/action/widget-action-dialog.component.html | 2 +- .../widget/data-key-config-dialog.component.html | 4 ++-- .../widget/lib/multiple-input-widget.component.html | 5 ++--- .../widget/lib/multiple-input-widget.component.ts | 9 ++++----- .../add-entities-to-customer-dialog.component.html | 6 +++--- .../dialogs/assign-to-customer-dialog.component.html | 6 +++--- .../home/pages/admin/general-settings.component.html | 4 ++-- .../modules/home/pages/admin/mail-server.component.html | 6 +++--- .../home/pages/admin/security-settings.component.html | 4 ++-- .../app/modules/home/pages/asset/asset.component.html | 2 +- .../modules/home/pages/customer/customer.component.html | 2 +- .../pages/dashboard/add-widget-dialog.component.html | 2 +- .../home/pages/dashboard/dashboard-form.component.html | 2 +- .../home/pages/dashboard/dashboard-page.component.html | 2 +- .../dashboard/dashboard-settings-dialog.component.html | 2 +- .../home/pages/dashboard/edit-widget.component.html | 2 +- .../home/pages/dashboard/edit-widget.component.ts | 2 -- .../manage-dashboard-layouts-dialog.component.html | 2 +- .../manage-dashboard-customers-dialog.component.html | 6 +++--- .../states/dashboard-state-dialog.component.html | 2 +- .../states/manage-dashboard-states-dialog.component.html | 2 +- .../device/device-credentials-dialog.component.html | 6 +++--- .../app/modules/home/pages/device/device.component.html | 2 +- .../home/pages/entity-view/entity-view.component.html | 2 +- .../pages/profile/change-password-dialog.component.html | 4 ++-- .../modules/home/pages/profile/profile.component.html | 4 ++-- .../pages/rulechain/add-rule-node-dialog.component.html | 4 ++-- .../rulechain/add-rule-node-link-dialog.component.html | 2 +- .../pages/rulechain/rule-node-details.component.html | 2 +- .../home/pages/rulechain/rule-node-details.component.ts | 2 -- .../home/pages/rulechain/rule-node-link.component.html | 2 +- .../home/pages/rulechain/rule-node-link.component.ts | 2 -- .../home/pages/rulechain/rulechain-page.component.html | 4 ++-- .../home/pages/rulechain/rulechain.component.html | 2 +- .../app/modules/home/pages/tenant/tenant.component.html | 2 +- .../modules/home/pages/user/add-user-dialog.component.ts | 6 +++--- .../src/app/modules/home/pages/user/user.component.html | 2 +- .../widget/save-widget-type-as-dialog.component.html | 6 +++--- .../home/pages/widget/widgets-bundle.component.html | 2 +- .../login/pages/login/create-password.component.html | 2 +- .../app/modules/login/pages/login/login.component.html | 2 +- .../pages/login/reset-password-request.component.html | 2 +- .../login/pages/login/reset-password.component.html | 2 +- .../dialog/node-script-test-dialog.component.html | 2 +- 57 files changed, 86 insertions(+), 97 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/alias/entity-alias-dialog.component.html b/ui-ngx/src/app/modules/home/components/alias/entity-alias-dialog.component.html index 0731de4ce2..b7f0ff21fc 100644 --- a/ui-ngx/src/app/modules/home/components/alias/entity-alias-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/alias/entity-alias-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ (isAdd ? 'alias.add' : 'alias.edit') | translate }}

diff --git a/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.html b/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.html index c0f22872eb..f817762f95 100644 --- a/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ title | translate }}

diff --git a/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html b/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html index ad85a91923..60fe5e30d7 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/add-widget-to-dashboard-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

attribute.add-widget-to-dashboard

diff --git a/ui-ngx/src/app/modules/home/components/attribute/edit-attribute-value-panel.component.html b/ui-ngx/src/app/modules/home/components/attribute/edit-attribute-value-panel.component.html index a3547cd486..8e2264fae1 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/edit-attribute-value-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/edit-attribute-value-panel.component.html @@ -15,8 +15,7 @@ limitations under the License. --> -
- +

layout.select

diff --git a/ui-ngx/src/app/modules/home/components/dashboard/select-target-state-dialog.component.html b/ui-ngx/src/app/modules/home/components/dashboard/select-target-state-dialog.component.html index 16d82ab9b2..41ba893e57 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard/select-target-state-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard/select-target-state-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

dashboard.select-state

diff --git a/ui-ngx/src/app/modules/home/components/details-panel.component.ts b/ui-ngx/src/app/modules/home/components/details-panel.component.ts index 66a9f3dc82..1e1ecb8a97 100644 --- a/ui-ngx/src/app/modules/home/components/details-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/details-panel.component.ts @@ -18,7 +18,7 @@ import { Component, EventEmitter, Input, Output } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { NgForm } from '@angular/forms'; +import { FormGroup, NgForm } from '@angular/forms'; @Component({ selector: 'tb-details-panel', @@ -33,14 +33,14 @@ export class DetailsPanelComponent extends PageComponent { @Input() isReadOnly = false; @Input() isAlwaysEdit = false; - theFormValue: NgForm; + theFormValue: FormGroup; @Input() - set theForm(value: NgForm) { + set theForm(value: FormGroup) { this.theFormValue = value; } - get theForm(): NgForm { + get theForm(): FormGroup { return this.theFormValue; } diff --git a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts index 2a39250872..871242c954 100644 --- a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.ts @@ -19,7 +19,7 @@ import { ErrorStateMatcher } from '@angular/material/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormControl, FormGroupDirective, NgForm } from '@angular/forms'; +import { FormControl, FormGroup, FormGroupDirective, NgForm } from '@angular/forms'; import { EntityTypeResource, EntityTypeTranslation } from '@shared/models/entity-type.models'; import { BaseData, HasId } from '@shared/models/base-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -40,7 +40,7 @@ export class AddEntityDialogComponent extends DialogComponent> implements OnInit, ErrorStateMatcher { entityComponent: EntityComponent>; - detailsForm: NgForm; + detailsForm: FormGroup; entitiesTableConfig: EntityTableConfig>; translations: EntityTypeTranslation; @@ -87,7 +87,7 @@ export class AddEntityDialogComponent extends const componentRef = viewContainerRef.createComponent(componentFactory, 0, injector); this.entityComponent = componentRef.instance; this.entityComponent.isEdit = true; - this.detailsForm = this.entityComponent.entityNgForm; + this.detailsForm = this.entityComponent.entityForm; } helpLinkId(): string { diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts index 37f63ecebc..af70020946 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts @@ -37,7 +37,7 @@ import { AppState } from '@core/core.state'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; import { BaseData, HasId } from '@shared/models/base-data'; import { EntityType, EntityTypeResource, EntityTypeTranslation } from '@shared/models/entity-type.models'; -import { NgForm } from '@angular/forms'; +import { FormGroup, NgForm } from '@angular/forms'; import { EntityComponent } from './entity.component'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { EntityAction } from '@home/models/entity/entity-component.models'; @@ -68,7 +68,7 @@ export class EntityDetailsPanelComponent extends PageComponent implements OnInit entityTabsComponentRef: ComponentRef>>; entityTabsComponent: EntityTabsComponent>; - detailsForm: NgForm; + detailsForm: FormGroup; entitiesTableConfigValue: EntityTableConfig>; isEditValue = false; @@ -189,7 +189,7 @@ export class EntityDetailsPanelComponent extends PageComponent implements OnInit this.entityComponentRef = viewContainerRef.createComponent(componentFactory, 0, injector); this.entityComponent = this.entityComponentRef.instance; this.entityComponent.isEdit = this.isEdit; - this.detailsForm = this.entityComponent.entityNgForm; + this.detailsForm = this.entityComponent.entityForm; this.subscriptions.push(this.entityComponent.entityAction.subscribe((action) => { this.entityAction.emit(action); })); diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts index 41ba7420dc..ef81cef161 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-tabs.component.ts @@ -30,7 +30,7 @@ import { AuditLogMode } from '@shared/models/audit-log.models'; import { DebugEventType, EventType } from '@shared/models/event.models'; import { AttributeScope, LatestTelemetry } from '@shared/models/telemetry/telemetry.models'; import { NULL_UUID } from '@shared/models/id/has-uuid'; -import { NgForm } from '@angular/forms'; +import { FormGroup, NgForm } from '@angular/forms'; import { PageLink } from '@shared/models/page/page-link'; @Directive() @@ -94,7 +94,7 @@ export abstract class EntityTabsComponent, } @Input() - detailsForm: NgForm; + detailsForm: FormGroup; private entityTabsSubject = new BehaviorSubject>(null); diff --git a/ui-ngx/src/app/modules/home/components/entity/entity.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity.component.ts index d98bcd3adf..b98f5a3b44 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity.component.ts @@ -35,8 +35,6 @@ export abstract class EntityComponent, entityForm: FormGroup; - @ViewChild('entityNgForm', {static: true}) entityNgForm: NgForm; - isEditValue: boolean; @Input() diff --git a/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.html b/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.html index 3e35485b9b..e53ff360ca 100644 --- a/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/import-export/import-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ importTitle }}

diff --git a/ui-ngx/src/app/modules/home/components/relation/relation-dialog.component.html b/ui-ngx/src/app/modules/home/components/relation/relation-dialog.component.html index 85f0ae64c5..2d637c4066 100644 --- a/ui-ngx/src/app/modules/home/components/relation/relation-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/relation/relation-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ (isAdd ? 'relation.add' : 'relation.edit' ) | translate }}

diff --git a/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html b/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html index b3d56b9774..7bf068a4d0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ (isAdd ? 'widget-config.add-action' : 'widget-config.edit-action' ) | translate }}

diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html index 4b6908bc6f..f05c97f3d5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ 'datakey.configuration' | translate }}

@@ -39,7 +39,7 @@
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/multiple-input-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/multiple-input-widget.component.ts index d3912e0a76..01adeacdce 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/multiple-input-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/multiple-input-widget.component.ts @@ -27,7 +27,7 @@ import { IWidgetSubscription } from '@core/api/widget-api.models'; import { isDefined, isEqual, isUndefined } from '@core/utils'; import { EntityType } from '@shared/models/entity-type.models'; import * as _moment from 'moment'; -import { FormBuilder, FormGroup, NgForm, ValidatorFn, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, ValidatorFn, Validators } from '@angular/forms'; import { RequestConfig } from '@core/http/http-utils'; import { AttributeService } from '@core/http/attribute.service'; import { AttributeData, AttributeScope, LatestTelemetry } from '@shared/models/telemetry/telemetry.models'; @@ -90,7 +90,6 @@ interface MultipleInputWidgetSource { export class MultipleInputWidgetComponent extends PageComponent implements OnInit, OnDestroy { @ViewChild('formContainer', {static: true}) formContainerRef: ElementRef; - @ViewChild('multipleInputForm', {static: true}) multipleInputForm: NgForm; @Input() ctx: WidgetContext; @@ -453,7 +452,7 @@ export class MultipleInputWidgetComponent extends PageComponent implements OnIni if (tasks.length) { forkJoin(tasks).subscribe( () => { - this.multipleInputForm.resetForm(); + this.multipleInputFormGroup.reset(undefined, {emitEvent: false}); this.multipleInputFormGroup.markAsPristine(); if (this.settings.showResultMessage) { this.ctx.showSuccessToast(this.translate.instant('widgets.input-widgets.update-successful'), @@ -467,13 +466,13 @@ export class MultipleInputWidgetComponent extends PageComponent implements OnIni } }); } else { - this.multipleInputForm.resetForm(); + this.multipleInputFormGroup.reset(undefined, {emitEvent: false}); this.multipleInputFormGroup.markAsPristine(); } } public discardAll() { - this.multipleInputForm.resetForm(); + this.multipleInputFormGroup.reset(undefined, {emitEvent: false}); this.sources.forEach((source) => { for (const key of this.visibleKeys(source)) { this.multipleInputFormGroup.get(key.formId).patchValue(key.value, {emitEvent: false}); diff --git a/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.html b/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.html index 9368219816..0b33828bc4 100644 --- a/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.html +++ b/ui-ngx/src/app/modules/home/dialogs/add-entities-to-customer-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ assignToCustomerTitle | translate }}

@@ -42,8 +42,8 @@
diff --git a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html index e87ccd3159..f2972687f5 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html @@ -28,7 +28,7 @@
- +
admin.mail-from @@ -97,10 +97,10 @@
-
diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index b0b8b52dd7..df2184967d 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -28,7 +28,7 @@
- +
@@ -140,7 +140,7 @@
-
diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 7a9b65d82e..a78d55ef93 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -62,7 +62,7 @@ [fxShow]="!isEdit && entity?.customerIsPublic && (assetScope === 'customer' || assetScope === 'tenant')"> {{ 'asset.asset-public' | translate }}
- +
asset.name diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer.component.html index a30e0acdd5..bb75d81fbd 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer.component.html @@ -58,7 +58,7 @@
- +
customer.title diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/add-widget-dialog.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/add-widget-dialog.component.html index 1b148c64ea..65f6900035 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/add-widget-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/add-widget-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

widget.add

diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html index 6589e4f81c..4c8376e218 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html @@ -90,7 +90,7 @@
- +
dashboard.title diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html index 76a59242a9..1c35a09282 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-page.component.html @@ -216,7 +216,7 @@ (closeDetails)="onEditWidgetClosed()" (toggleDetailsEditMode)="onRevertWidgetEdit()" (applyDetails)="saveWidget()" - [theForm]="tbEditWidget.widgetForm"> + [theForm]="tbEditWidget.widgetFormGroup">
diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-settings-dialog.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-settings-dialog.component.html index 0227668842..41a3151d13 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-settings-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-settings-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{settings ? 'dashboard.settings' : 'layout.settings'}}

diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.html index 110fec0b3f..6dba30e6f2 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/edit-widget.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +
- +

layout.manage

diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html index 45072cda33..1e6c21cb7c 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -

{{ titleText | translate }}

@@ -43,8 +43,8 @@ diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html index 563dc43354..edd35ba06e 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

rulenode.add

@@ -42,7 +42,7 @@