From 69c42d243bfc62baef2cedf5f1b3bc81c8fda97e Mon Sep 17 00:00:00 2001 From: nordmif Date: Tue, 14 Apr 2020 14:15:57 +0300 Subject: [PATCH 01/39] changed device attributes name to clarify its functionality --- .../thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 653b571b4b..03f0652c3f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "device attributes", + name = "related device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + From e8b94d566cff1f7c85d6707370a535041f420da2 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 15 Apr 2020 16:57:40 +0300 Subject: [PATCH 02/39] init commit --- ...rvice.java => AbstractCleanUpService.java} | 40 ++++--------- .../ttl/events/EventsCleanUpService.java | 57 +++++++++++++++++++ .../AbstractTimeseriesCleanUpService.java | 49 ++++++++++++++++ .../PsqlTimeseriesCleanUpService.java | 2 +- .../TimescaleTimeseriesCleanUpService.java | 2 +- .../src/main/resources/thingsboard.yml | 11 +++- 6 files changed, 127 insertions(+), 34 deletions(-) rename application/src/main/java/org/thingsboard/server/service/ttl/{AbstractTimeseriesCleanUpService.java => AbstractCleanUpService.java} (67%) create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/PsqlTimeseriesCleanUpService.java (96%) rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/TimescaleTimeseriesCleanUpService.java (95%) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 2399450d86..61d81ae0b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlDao; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService { - - @Value("${sql.ttl.ts_key_value_ttl}") - protected long systemTtl; - @Value("${sql.ttl.enabled}") - private boolean ttlTaskExecutionEnabled; +@Slf4j +@PsqlDao +public abstract class AbstractCleanUpService { @Value("${spring.datasource.url}") - private String dbUrl; + protected String dbUrl; @Value("${spring.datasource.username}") - private String dbUserName; + protected String dbUserName; @Value("${spring.datasource.password}") - private String dbPassword; - - @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - protected abstract void doCleanUp(Connection connection); + protected String dbPassword; protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService { return removed; } - private void getWarnings(Statement statement) throws SQLException { + protected void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService { } } -} \ No newline at end of file + protected abstract void doCleanUp(Connection connection); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java new file mode 100644 index 0000000000..bb7d112bb1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.events; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlDao +@Slf4j +@Service +public class EventsCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.events.events_key_value_ttl}") + private long ttl; + + @Value("${sql.ttl.events.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + @Override + protected void doCleanUp(Connection connection) { + log.info("ttl: [{}]", ttl); + log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); + // TODO: 4/15/20 Do a clean up. + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java new file mode 100644 index 0000000000..75b07b9176 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlTsAnyDao +@Slf4j +public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.ts.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.ts.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index ab344dc518..cd403ee3b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index 1dbdb4ad55..f5898b9b20 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0402c91387..9283f9283c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -217,9 +217,14 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - enabled: "${SQL_TTL_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds - ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds + ts: + enabled: "${SQL_TTL_TS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds + events: + enabled: "${SQL_TTL_EVENTS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: From adae8aeb30631d292573833d58d621ef08606a5a Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Fri, 17 Apr 2020 16:55:17 +0300 Subject: [PATCH 03/39] Improved implementaton of ProcessingAttemptContext to avoid synchronizations --- .../queue/ProcessingAttemptContext.java | 21 +++----- .../queue/ProcessingAttemptContextTest.java | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java index 2073ff8c21..aefb1697cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -27,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ProcessingAttemptContext { private final TbRuleEngineSubmitStrategy submitStrategy; + private final AtomicInteger pendingCount; private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); @Getter private final ConcurrentMap> pendingMap; @@ -45,6 +47,7 @@ public class ProcessingAttemptContext { public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); } public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { @@ -54,16 +57,12 @@ public class ProcessingAttemptContext { public void onSuccess(UUID id) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; successMap.put(id, msg); + submitStrategy.onSuccess(id); } - submitStrategy.onSuccess(id); if (empty) { processingTimeoutLatch.countDown(); } @@ -72,13 +71,9 @@ public class ProcessingAttemptContext { public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; failedMap.put(id, msg); exceptionsMap.putIfAbsent(tenantId, e); } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java new file mode 100644 index 0000000000..6c3914abc8 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -0,0 +1,51 @@ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ProcessingAttemptContextTest { + + @Test + public void testHighConcurrencyCase() throws InterruptedException { + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + int msgCount = 1000; + int parallelCount = 5; + ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); + try { + ConcurrentMap> messages = new ConcurrentHashMap<>(); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + when(strategyMock.getPendingMap()).thenReturn(messages); + ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + for (UUID uuid : messages.keySet()) { + for (int i = 0; i < parallelCount; i++) { + executorService.submit(() -> context.onSuccess(uuid)); + } + } + Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); + Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); + } finally { + executorService.shutdownNow(); + } + } +} From eae9d3c8e29ac276a1005c960951ac40535e6ad2 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Mon, 20 Apr 2020 10:55:25 +0300 Subject: [PATCH 04/39] Processing Atempt Context Improvement --- .../queue/ProcessingAttemptContextTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java index 6c3914abc8..2de2414f71 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; From b1f87206f3d930fbec7270ea018ba68a188315fc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 11:02:40 +0300 Subject: [PATCH 05/39] Minor improvements --- .../server/actors/ruleChain/DefaultTbContext.java | 3 +++ .../server/actors/ruleChain/RuleNodeActor.java | 2 ++ .../server/service/state/DefaultDeviceStateService.java | 8 ++++---- .../server/service/state/DeviceStateService.java | 2 +- .../common/transport/service/DefaultTransportService.java | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b268236448..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -192,6 +192,9 @@ class DefaultTbContext implements TbContext { @Override public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } tbMsg.getCallback().onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 64431fdf33..abb98468bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -54,6 +55,7 @@ public class RuleNodeActor extends ComponentActor> partitionedDevices = new ConcurrentHashMap<>(); - private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 16c699b38a..1665a27b84 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener callback.onSuccess(null)); } } From 341f0d14ab0ebe221b1ff2ea94bfa425980c03ab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 12:05:27 +0300 Subject: [PATCH 06/39] Fixed tests inconsistency due to new queues --- .../mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java | 4 ++++ .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 5ba4dfcbae..485d618a01 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -109,6 +109,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -165,6 +167,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 122bef2e26..7e329d8b05 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,7 +79,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(1000); + Thread.sleep(2000); List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); From 58d9c313a89d3929ad7c3d0e46ed287b91132355 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 16:12:03 +0300 Subject: [PATCH 07/39] Fixed tests --- .../DefaultRuleEngineStatisticsService.java | 11 +- .../controller/BaseAssetControllerTest.java | 128 ++++++++++-------- .../BaseEntityViewControllerTest.java | 6 +- .../controller/ControllerNoSqlTestSuite.java | 7 + .../controller/ControllerSqlTestSuite.java | 7 + .../server/mqtt/MqttNoSqlTestSuite.java | 7 + .../server/mqtt/MqttSqlTestSuite.java | 7 + .../rules/RuleEngineNoSqlTestSuite.java | 7 + .../server/rules/RuleEngineSqlTestSuite.java | 7 + .../server/system/SystemNoSqlTestSuite.java | 7 + .../server/system/SystemSqlTestSuite.java | 7 + .../server/queue/memory/InMemoryStorage.java | 42 +++--- .../queue/memory/InMemoryTbQueueConsumer.java | 11 +- tools/pom.xml | 2 +- 14 files changed, 173 insertions(+), 83 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 3951703351..a506b87ab0 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -47,6 +47,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { + public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final FutureCallback CALLBACK = new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { @@ -95,7 +96,13 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + try { + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + } catch (DataValidationException e2) { + if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e2; + } + } }); ruleEngineStats.reset(); } @@ -113,7 +120,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS asset = new Asset(); asset.setTenantId(tenantId); asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType("TbServiceQueue"); + asset.setType(TB_SERVICE_QUEUE); asset = assetService.saveAsset(asset); } assetId = asset.getId(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 56c73b5e13..0422a85416 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/"+savedAsset.getId().getId().toString()) + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/"+savedAsset.getId().getId().toString()) + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); + Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index e60f549610..420446e04d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -426,7 +426,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); -// client.disconnect(); + client.disconnect(); } private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { @@ -463,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - + client.disconnect(); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java index 4ad41824a0..781c483fc5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.controller; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false), new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 8dc0acff57..347eaee7eb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.controller; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index 9c4030cff4..7360c5c506 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 2863589ba1..70de3b27ca 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.mqtt; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -32,4 +34,9 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java index e44c383452..fbab13147c 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java @@ -16,11 +16,13 @@ package org.thingsboard.server.rules; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 5f930821f7..83ac5f7703 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.rules; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java index 41be7641c6..c4182db3ee 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.system; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index b12d513ce0..6060a2cc2b 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.system; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,4 +37,9 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index f919df871e..e119d1433b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -30,11 +30,9 @@ import java.util.concurrent.TimeUnit; public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; - private volatile boolean stopped; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); - stopped = false; } public static InMemoryStorage getInstance() { @@ -52,33 +50,31 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) { + public List get(String topic, long durationInMillis) throws InterruptedException { if (storage.containsKey(topic)) { - try { - List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); - if (first != null) { - entities = new ArrayList<>(); - entities.add(first); - List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 999); - for (TbQueueMsg other : otherList) { - entities.add((T) other); - } - } else { - entities = Collections.emptyList(); - } - return entities; - } catch (InterruptedException e) { - if (!stopped) { - log.warn("Queue was interrupted", e); + List entities; + T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 999); + for (TbQueueMsg other : otherList) { + entities.add((T) other); } + } else { + entities = Collections.emptyList(); } + return entities; } return Collections.emptyList(); } - public void stop() { - stopped = true; + /** + * Used primarily for testing. + */ + public void cleanup() { + storage.clear(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 2b81839540..3920f143f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -66,7 +66,16 @@ public class InMemoryTbQueueConsumer implements TbQueueCon if (subscribed) { List messages = partitions .stream() - .map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) + .map(tpi -> { + try { + return storage.get(tpi.getFullTopicName(), durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Queue was interrupted.", e); + } + return Collections.emptyList(); + } + }) .flatMap(List::stream) .map(msg -> (T) msg).collect(Collectors.toList()); if (messages.size() > 0) { diff --git a/tools/pom.xml b/tools/pom.xml index 64027c478e..1960d330d0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.4 + 3.11.6 com.datastax.cassandra From 149ec2804b410ab7e099154f139f55c01e6ae696 Mon Sep 17 00:00:00 2001 From: Ilya Barkov Date: Mon, 20 Apr 2020 16:20:33 +0300 Subject: [PATCH 08/39] Feature/gateway remote configuration (#2627) * new widget for gateway with the last telemetry type * add isStateForm for the last telemetry type * add config update save * update thingboard yaml * remove update thingboard yaml * add new widget to gateway_widgets.json * delete state gateway * delete state gateway * Revert "delete state gateway" This reverts commit d8024ead * change yml * edit name isReadOnly * fix bug isReadOnly * [Gateway remote configuration] Changed default logs.conf file Co-authored-by: nick --- .../widget_bundles/gateway_widgets.json | 32 ++++++++ ui/src/app/common/types.constant.js | 18 ++++- .../gateway/gateway-config.directive.js | 7 +- .../gateway/gateway-config.tpl.html | 12 +-- .../gateway/gateway-form.directive.js | 80 ++++++++++++------- .../components/gateway/gateway-form.tpl.html | 63 ++++++++------- ui/src/app/locale/locale.constant-en_US.json | 7 +- 7 files changed, 153 insertions(+), 66 deletions(-) diff --git a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json index 0158b38fe9..7d5dd46086 100644 --- a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json @@ -5,6 +5,38 @@ "image": null }, "widgetTypes": [ + { + "alias": "attributes_card", + "name": "Gateway events", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 8, + "resources": [], + "templateHtml": "", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "let types;\nlet eventsReg = \"eventsReg\";\n\nself.onInit = function() {\n \n self.ctx.datasourceTitleCells = [];\n self.ctx.valueCells = [];\n self.ctx.labelCells = [];\n types = self.ctx.$scope.$injector.get('types');\n \n if (self.ctx.datasources.length && self.ctx.datasources[0].type === types.datasourceType.entity) {\n getDatasourceKeys(self.ctx.datasources[0]);\n } else {\n processDatasources(self.ctx.datasources);\n }\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.valueCells.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData && cellData.data && cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length -\n 1];\n var value = tvPair[1];\n var textValue;\n //toDo -> + IsNumber\n \n if (isNumber(value)) {\n var decimals = self.ctx.decimals;\n var units = self.ctx.units;\n if (cellData.dataKey.decimals || cellData.dataKey.decimals === 0) {\n decimals = cellData.dataKey.decimals;\n }\n if (cellData.dataKey.units) {\n units = cellData.dataKey.units;\n }\n txtValue = self.ctx.utils.formatValue(value, decimals, units, false);\n }\n else {\n txtValue = value;\n }\n self.ctx.valueCells[i].html(txtValue);\n }\n }\n \n function isNumber(n) {\n return !isNaN(parseFloat(n)) && isFinite(n);\n }\n}\n\nself.onResize = function() {\n var datasourceTitleFontSize = self.ctx.height/8;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n datasourceTitleFontSize = self.ctx.width/12;\n }\n datasourceTitleFontSize = Math.min(datasourceTitleFontSize, 20);\n for (var i = 0; i < self.ctx.datasourceTitleCells.length; i++) {\n self.ctx.datasourceTitleCells[i].css('font-size', datasourceTitleFontSize+'px');\n }\n var valueFontSize = self.ctx.height/9;\n var labelFontSize = self.ctx.height/9;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n valueFontSize = self.ctx.width/15;\n labelFontSize = self.ctx.width/15;\n }\n valueFontSize = Math.min(valueFontSize, 18);\n labelFontSize = Math.min(labelFontSize, 18);\n\n for (i = 0; i < self.ctx.valueCells; i++) {\n self.ctx.valueCells[i].css('font-size', valueFontSize+'px');\n self.ctx.valueCells[i].css('height', valueFontSize*2.5+'px');\n self.ctx.valueCells[i].css('padding', '0px ' + valueFontSize + 'px');\n self.ctx.labelCells[i].css('font-size', labelFontSize+'px');\n self.ctx.labelCells[i].css('height', labelFontSize*2.5+'px');\n self.ctx.labelCells[i].css('padding', '0px ' + labelFontSize + 'px');\n } \n}\n\nfunction processDatasources(datasources) {\n var i = 0;\n var tbDatasource = datasources[i];\n var datasourceId = 'tbDatasource' + i;\n self.ctx.$container.append(\n \"
\"\n );\n\n var datasourceContainer = $('#' + datasourceId,\n self.ctx.$container);\n\n datasourceContainer.append(\n \"
\" +\n tbDatasource.name + \"
\"\n );\n \n var datasourceTitleCell = $('.tbDatasource-title', datasourceContainer);\n self.ctx.datasourceTitleCells.push(datasourceTitleCell);\n \n var tableId = 'table' + i;\n datasourceContainer.append(\n \"
\"\n );\n var table = $('#' + tableId, self.ctx.$container);\n\n for (var a = 0; a < tbDatasource.dataKeys.length; a++) {\n var dataKey = tbDatasource.dataKeys[a];\n var labelCellId = 'labelCell' + a;\n var cellId = 'cell' + a;\n table.append(\"\" + dataKey.label +\n \"\");\n var labelCell = $('#' + labelCellId, table);\n self.ctx.labelCells.push(labelCell);\n var valueCell = $('#' + cellId, table);\n self.ctx.valueCells.push(valueCell);\n }\n self.onResize();\n}\n\nfunction getDatasourceKeys (datasource) {\n let attributeService = self.ctx.$scope.$injector.get('attributeService');\n if (datasource.entityId && datasource.entityType) {\n attributeService.getEntityKeys(datasource.entityType, datasource.entityId, \"\" , types.dataKeyType.timeseries).then(\n function(data){\n if (data.length) {\n subscribeForKeys (datasource, data);\n }\n });\n }\n}\n\nfunction subscribeForKeys (datasource, data) {\n let eventsRegVals = self.ctx.settings[eventsReg];\n if (eventsRegVals && eventsRegVals.length > 0) {\n var dataKeys = [];\n data.sort();\n data.forEach(dataValue => {eventsRegVals.forEach(event => {\n if (dataValue.toLowerCase().includes(event.toLowerCase())) {\n var dataKey = {\n type: types.dataKeyType.timeseries,\n name: dataValue,\n label: dataValue,\n settings: {},\n _hash: Math.random()\n };\n dataKeys.push(dataKey);\n }\n })});\n\n if (dataKeys.length) {\n updateSubscription (datasource, dataKeys);\n }\n }\n}\n\nfunction updateSubscription (datasource, dataKeys) {\n var datasources = [\n {\n type: types.datasourceType.entity,\n name: datasource.aliasName,\n aliasName: datasource.aliasName,\n entityAliasId: datasource.entityAliasId,\n dataKeys: dataKeys\n }\n ];\n \n var subscriptionOptions = {\n datasources: datasources,\n useDashboardTimewindow: false,\n type: types.widgetType.latest.value,\n callbacks: {\n onDataUpdated: (subscription) => {\n self.ctx.data = subscription.data;\n self.onDataUpdated();\n }\n }\n };\n \n processDatasources(datasources);\n self.ctx.subscriptionApi.createSubscription(subscriptionOptions, true).then(\n (subscription) => {\n self.ctx.defaultSubscription = subscription;\n }\n );\n}\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\n dataKeysOptional: true\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayEventsForm\",\n \"properties\": {\n \"eventsTitle\": {\n \"title\": \"Gateway events form title\",\n \"type\": \"string\",\n \"default\": \"Gateway Events Form\"\n },\n \"eventsReg\": {\n \"title\": \"Events filten.\",\n \"type\": \"array\",\n \"items\": {\n \"title\": \"Event key contains\",\n \"type\": \"string\"\n }\n }\n }\n },\n \"form\": [\n \"eventsTitle\",\n \"eventsReg\"\n ]\n}", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Function Math.round\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.826503672916844,\"funcBody\":\"return Math.round(1000*Math.sin(time/5000));\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"eventsTitle\":\"Gateway Events Form\",\"eventsReg\":[]},\"title\":\"Gateway events\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" + } + }, + { + "alias": "config_form_latest", + "name": "Gateway configuration (Single device)", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 9, + "resources": [], + "templateHtml": "\n", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n var id = self.ctx.$scope.$injector.get('utils').guid();\n scope.formId = \"form-\"+id;\n scope.ctx = self.ctx;\n scope.isStateForm = true;\n}\n\nself.onResize = function() {\n self.ctx.$scope.$broadcast('gateway-form-resize', self.ctx.$scope.formId);\n}\n\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\t\t\n dataKeysOptional: true\t\t\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayConfigForm\",\n \"properties\": {\n \"gatewayTitle\": {\n \"title\": \"Gateway form\",\n \"type\": \"string\",\n \"default\": \"Gateway configuration (Single device)\"\n },\n \"readOnly\": {\n \"title\": \"Read Only\",\n \"type\": \"boolean\",\n \"default\": false\n }\n }\n },\n \"form\": [\n \"gatewayTitle\",\n \"readOnly\"\n ]\n}\n", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Gateway configuration (Single device)\"}" + } + }, { "alias": "extension_configuration_widget", "name": "Extensions table", diff --git a/ui/src/app/common/types.constant.js b/ui/src/app/common/types.constant.js index aff3821573..8285430c56 100644 --- a/ui/src/app/common/types.constant.js +++ b/ui/src/app/common/types.constant.js @@ -606,13 +606,29 @@ export default angular.module('thingsboard.types', []) value: "modbus", name: "Modbus" }, - opc_ua: { + opcua: { value: "opcua", name: "OPC-UA" }, ble: { value: "ble", name: "BLE" + }, + request: { + value: "request", + name: "Request" + }, + can: { + value: "can", + name: "CAN" + }, + bacnet: { + value: "bacnet", + name: "BACnet" + }, + custom: { + value: "custom", + name: "Custom" } }, gatewayLogLevel: { diff --git a/ui/src/app/components/gateway/gateway-config.directive.js b/ui/src/app/components/gateway/gateway-config.directive.js index 120a6d3e15..19e7f554b9 100644 --- a/ui/src/app/components/gateway/gateway-config.directive.js +++ b/ui/src/app/components/gateway/gateway-config.directive.js @@ -37,7 +37,8 @@ function GatewayConfig() { disabled: '=ngDisabled', gatewayConfig: '=', changeAlignment: '=', - theForm: '=' + theForm: '=', + isReadOnly: '=' }, controller: GatewayConfigController, controllerAs: 'vm', @@ -83,6 +84,10 @@ function GatewayConfigController($scope, $document, $mdDialog, $mdUtil, $window, multiple: true, }).then(function (config) { if (config && index > -1) { + console.log(config); //eslint-disable-line + if (!angular.equals(vm.gatewayConfig[index].config, config)) { + $scope.gatewayConfiguration.$setDirty(); + } vm.gatewayConfig[index].config = config; } }); diff --git a/ui/src/app/components/gateway/gateway-config.tpl.html b/ui/src/app/components/gateway/gateway-config.tpl.html index eef7c737e4..e4525b2bf2 100644 --- a/ui/src/app/components/gateway/gateway-config.tpl.html +++ b/ui/src/app/components/gateway/gateway-config.tpl.html @@ -18,7 +18,7 @@
-
@@ -26,7 +26,7 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - @@ -39,7 +39,7 @@ -
@@ -49,7 +49,7 @@
- more_horiz @@ -57,7 +57,7 @@ {{ 'gateway.update-config' | translate }} - close @@ -70,7 +70,7 @@ layout-align="center center" ng-class="{'disabled': vm.disabled}" class="no-data-found" translate>{{'gateway.no-connectors'}}
- {{ 'gateway.connector-add' | translate }} diff --git a/ui/src/app/components/gateway/gateway-form.directive.js b/ui/src/app/components/gateway/gateway-form.directive.js index 5efa2ee121..436bc6bcd5 100644 --- a/ui/src/app/components/gateway/gateway-form.directive.js +++ b/ui/src/app/components/gateway/gateway-form.directive.js @@ -31,11 +31,15 @@ function GatewayForm() { scope: true, bindToController: { formId: '=', - ctx: '=' + ctx: '=', + isStateForm: '=', + deviceName: '=', + isReadOnly: '=', + isState: '=' }, controller: GatewayFormController, controllerAs: 'vm', - templateUrl: gatewayFormTemplate + templateUrl: gatewayFormTemplate, }; } @@ -47,10 +51,11 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, const configurationAttribute = "configuration"; const remoteLoggingLevelAttribute = "RemoteLoggingLevel"; - const templateLogsConfig = '[loggers]}}keys=root, service, connector, converter, tb_connection, storage, extension}}[handlers]}}keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler}}[formatters]}}keys=LogFormatter}}[logger_root]}}level=ERROR}}handlers=consoleHandler}}[logger_connector]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=connector}}[logger_storage]}}level={ERROR}}}handlers=storageHandler}}formatter=LogFormatter}}qualname=storage}}[logger_tb_connection]}}level={ERROR}}}handlers=tb_connectionHandler}}formatter=LogFormatter}}qualname=tb_connection}}[logger_service]}}level={ERROR}}}handlers=serviceHandler}}formatter=LogFormatter}}qualname=service}}[logger_converter]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=converter}}[logger_extension]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=extension}}[handler_consoleHandler]}}class=StreamHandler}}level={ERROR}}}formatter=LogFormatter}}args=(sys.stdout,)}}[handler_connectorHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}connector.log", "d", 1, 7,)}}[handler_storageHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}storage.log", "d", 1, 7,)}}[handler_serviceHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}service.log", "d", 1, 7,)}}[handler_converterHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}converter.log", "d", 1, 3,)}}[handler_extensionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}extension.log", "d", 1, 3,)}}[handler_tb_connectionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}tb_connection.log", "d", 1, 3,)}}[formatter_LogFormatter]}}format="%(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s" }}datefmt="%Y-%m-%d %H:%M:%S"'; + const templateLogsConfig = '[loggers]}}keys=root, service, connector, converter, tb_connection, storage, extension}}[handlers]}}keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler}}[formatters]}}keys=LogFormatter}}[logger_root]}}level=ERROR}}handlers=consoleHandler}}[logger_connector]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=connector}}[logger_storage]}}level={ERROR}}}handlers=storageHandler}}formatter=LogFormatter}}qualname=storage}}[logger_tb_connection]}}level={ERROR}}}handlers=tb_connectionHandler}}formatter=LogFormatter}}qualname=tb_connection}}[logger_service]}}level={ERROR}}}handlers=serviceHandler}}formatter=LogFormatter}}qualname=service}}[logger_converter]}}level={ERROR}}}handlers=converterHandler}}formatter=LogFormatter}}qualname=converter}}[logger_extension]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=extension}}[handler_consoleHandler]}}class=StreamHandler}}level={ERROR}}}formatter=LogFormatter}}args=(sys.stdout,)}}[handler_connectorHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}connector.log", "d", 1, 7,)}}[handler_storageHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}storage.log", "d", 1, 7,)}}[handler_serviceHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}service.log", "d", 1, 7,)}}[handler_converterHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}converter.log", "d", 1, 3,)}}[handler_extensionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}extension.log", "d", 1, 3,)}}[handler_tb_connectionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}tb_connection.log", "d", 1, 3,)}}[formatter_LogFormatter]}}format="%(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s" }}datefmt="%Y-%m-%d %H:%M:%S"'; vm.types = types; - + vm.deviceNameForm = (vm.deviceName) ? vm.deviceName : null; + vm.idForm = Math.random().toString(36).replace(/^0\.[0-9]*/, ''); vm.configurations = { gateway: '', host: $document[0].domain, @@ -74,7 +79,6 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, let archiveFileName = ''; let gatewayNameExists = ''; let successfulSaved = ''; - vm.securityTypes = [{ name: 'gateway.security-types.access-token', value: 'accessToken' @@ -92,13 +96,18 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, }]; $scope.$watch('vm.ctx', function () { - if (vm.ctx ) { + if (vm.ctx) { + vm.isStateForm = $scope.isStateForm; vm.settings = vm.ctx.settings; vm.widgetConfig = vm.ctx.widgetConfig; - initializeConfig(); + if (vm.ctx.datasources && vm.ctx.datasources.length) { + vm.deviceNameForm = vm.ctx.datasources[0].name; + } } + initializeConfig(); }); + $scope.$on('gateway-form-resize', function (event, formId) { if (vm.formId == formId) { updateWidgetDisplaying(); @@ -106,21 +115,34 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, }); function updateWidgetDisplaying() { - vm.changeAlignment = (vm.ctx.$container[0].offsetWidth <= 425); + if (vm.ctx) { + vm.changeAlignment = (vm.ctx.$container[0].offsetWidth <= 425); + } } function initWidgetSettings() { let widgetTitle; - if (vm.settings.widgetTitle && vm.settings.widgetTitle.length) { - widgetTitle = utils.customTranslation(vm.settings.widgetTitle, vm.settings.widgetTitle); + if (vm.settings) { + vm.isReadOnlyForm = (vm.settings.readOnly) ? vm.settings.readOnly : false; + if (vm.settings.gatewayTitle && vm.settings.gatewayTitle.length) { + widgetTitle = utils.customTranslation(vm.settings.gatewayTitle, vm.settings.gatewayTitle); + } } else { + vm.isReadOnlyForm = false; widgetTitle = $translate.instant('gateway.gateway'); } - vm.ctx.widgetTitle = widgetTitle; + if (vm.ctx) { + vm.ctx.widgetTitle = widgetTitle; + } - archiveFileName = vm.settings.archiveFileName && vm.settings.archiveFileName.length ? vm.settings.archiveFileName : 'gatewayConfiguration'; - gatewayNameExists = utils.customTranslation(vm.settings.gatewayNameExists, vm.settings.gatewayNameExists) || $translate.instant('gateway.gateway-exists'); - successfulSaved = utils.customTranslation(vm.settings.successfulSave, vm.settings.successfulSave) || $translate.instant('gateway.gateway-saved'); + archiveFileName = vm.settings && vm.settings.archiveFileName && vm.settings.archiveFileName.length ? vm.settings.archiveFileName : 'gatewayConfiguration'; + if (vm.settings) { + gatewayNameExists = utils.customTranslation(vm.settings.deviceNameExist, vm.settings.deviceNameExist) || $translate.instant('gateway.gateway-exists'); + successfulSaved = utils.customTranslation(vm.settings.successfulSave, vm.settings.successfulSave) || $translate.instant('gateway.gateway-saved'); + } else { + gatewayNameExists = $translate.instant('gateway.gateway-exists'); + successfulSaved = $translate.instant('gateway.gateway-saved'); + } } function initializeConfig() { @@ -129,6 +151,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, getGatewaysList(true); } + vm.getAccessToken = (deviceId) => { if (deviceId.id) { getDeviceCredentials(deviceId.id); @@ -152,15 +175,15 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, deviceService.findByName(deviceObj.name, {ignoreErrors: true}) .then( function () { - toast.showError(gatewayNameExists, angular.element('.gateway-form'),'top left'); + toast.showError(gatewayNameExists, angular.element('.gateway-form'), 'top left'); }, function () { - if(vm.settings.gatewayType && vm.settings.gatewayType.length){ + if (vm.settings.gatewayType && vm.settings.gatewayType.length) { deviceObj.type = vm.settings.gatewayType; } deviceService.saveDevice(deviceObj).then( (device) => { - getDeviceCredentials(device.id.id).then(() =>{ + getDeviceCredentials(device.id.id).then(() => { getGatewaysList(); }); } @@ -173,8 +196,8 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, saveAttribute(configurationAttribute, $window.btoa(angular.toJson(getGatewayConfigJSON())), types.attributesScope.shared.value), saveAttribute(configurationDraftsAttribute, $window.btoa(angular.toJson(getDraftConnectorJSON())), types.attributesScope.server.value), saveAttribute(remoteLoggingLevelAttribute, vm.configurations.remoteLoggingLevel.toUpperCase(), types.attributesScope.shared.value) - ]).then(() =>{ - toast.showSuccess(successfulSaved, 2000, angular.element('.gateway-form'),'top left'); + ]).then(() => { + toast.showSuccess(successfulSaved, 2000, angular.element('.gateway-form'), 'top left'); }) }; @@ -238,7 +261,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, config += ' max_records_per_file: ' + vm.configurations.maxRecordsCount + '\n'; } config += 'connectors:\n'; - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { config += ' -\n'; config += ' name: ' + vm.configurations.connectors[i].name + '\n'; @@ -250,7 +273,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, } function generateConfigConnectorFiles(fileZipAdd) { - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { fileZipAdd[generateFileName(vm.configurations.connectors[i].name)] = angular.toJson(vm.configurations.connectors[i].config); } @@ -276,7 +299,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, function gatewayMainConfigJSON() { let configuration = {}; - + let thingsBoard = {}; thingsBoard.host = vm.configurations.host; thingsBoard.remoteConfiguration = vm.configurations.remoteConfiguration; @@ -325,10 +348,10 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, } function gatewayConnectorConfigJSON(gatewayConfiguration) { - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { let typeConnector = vm.configurations.connectors[i].configType; - if(!angular.isArray(gatewayConfiguration[typeConnector])){ + if (!angular.isArray(gatewayConfiguration[typeConnector])) { gatewayConfiguration[typeConnector] = []; } @@ -343,7 +366,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, function getDraftConnectorJSON() { let draftConnector = {}; - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (!vm.configurations.connectors[i].enabled) { let connector = { connector: vm.configurations.connectors[i].configType, @@ -362,7 +385,10 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, const device = devices[i]; if (device.additionalInfo !== null && device.additionalInfo.gateway === true) { vm.gateways.push(device); - if (firstInit && vm.gateways.length && device.name === vm.gateways[0].name) { + if (vm.deviceNameForm && firstInit && vm.gateways.length && device.name === vm.deviceNameForm) { + vm.configurations.gateway = device; + vm.getAccessToken(device.id); + } else if (firstInit && vm.gateways.length && device.name === vm.gateways[0].name) { vm.configurations.gateway = device; vm.getAccessToken(device.id); } @@ -385,7 +411,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, for (let connectorType in keyValue) { let name = "No name"; if (Object.prototype.hasOwnProperty.call(keyValue[connectorType], 'name')) { - name = keyValue[connectorType].name ; + name = keyValue[connectorType].name; } let connector = { enabled: true, diff --git a/ui/src/app/components/gateway/gateway-form.tpl.html b/ui/src/app/components/gateway/gateway-form.tpl.html index abe030d543..c8d44d1fcb 100644 --- a/ui/src/app/components/gateway/gateway-form.tpl.html +++ b/ui/src/app/components/gateway/gateway-form.tpl.html @@ -17,30 +17,36 @@ -->
- - + +
{{ 'gateway.thingsboard' | translate | uppercase }}
 - +
{{ 'gateway.thingsboard' | translate | uppercase }}
 - + + + + - + {{securityType.name | translate}} @@ -50,14 +56,14 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - +
gateway.thingsboard-host-required
 -
gateway.thingsboard-port-required
@@ -70,18 +76,18 @@
- + - + - +
- {{ 'gateway.remote' | translate }} @@ -90,7 +96,7 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - + {{logLevel}} @@ -99,7 +105,7 @@ -
gateway.path-logs-required
@@ -109,14 +115,14 @@ - +
{{ 'gateway.storage' | translate | uppercase }}
 - +
{{ 'gateway.storage' | translate | uppercase }}
@@ -124,18 +130,17 @@ - + {{storageType.name | translate}} -
-
gateway.storage-pack-size-required
@@ -143,10 +148,9 @@
gateway.storage-pack-size-pattern
 - -
gateway.storage-max-records-required
@@ -155,13 +159,12 @@
-
-
gateway.storage-max-files-required
@@ -169,10 +172,9 @@
gateway.storage-max-files-pattern
 - -
gateway.storage-path-required
@@ -182,14 +184,14 @@ - +
{{ 'gateway.connectors' | translate | uppercase }}
 - +
{{ 'gateway.connectors' | translate | uppercase }}
@@ -198,13 +200,14 @@ + change-alignment="vm.changeAlignment" + is-read-only="vm.isReadOnlyForm"> -
+
{{'gateway.download-tip' | translate }} - {{'action.save' | translate }} - {{'gateway.save-tip' | translate }} + {{'gateway.save-tip' | translate }} +
diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index d69778a787..8c7a89580d 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1201,7 +1201,12 @@ "tls-path-private-key": "Path to private key on gateway", "toggle-fullscreen": "Toggle fullscreen", "transformer-json-config": "Configuration JSON*", - "update-config": "Add/update configuration JSON" + "update-config": "Add/update configuration JSON", + "state-title": "Gateway state", + "show-config-tip": "Show gateway configuration", + "title-show-config": "Show gateway configuration", + "read-only": "Read only", + "read-write": "" }, "grid": { "delete-item-title": "Are you sure you want to delete this item?", From 1b9df18c4592ea16d35b0bf0eace8e1f20a2ca43 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 20:30:56 +0300 Subject: [PATCH 09/39] created TbPubSubSubscriptionSettings --- .../src/main/resources/thingsboard.yml | 7 +- .../provider/PubSubMonolithQueueFactory.java | 40 +++++++---- .../provider/PubSubTbCoreQueueFactory.java | 36 ++++++---- .../PubSubTbRuleEngineQueueFactory.java | 32 ++++++--- .../provider/PubSubTransportQueueFactory.java | 36 ++++++---- .../server/queue/pubsub/TbPubSubAdmin.java | 35 +++++++-- .../server/queue/pubsub/TbPubSubSettings.java | 3 - .../pubsub/TbPubSubSubscriptionSettings.java | 71 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 7 +- .../src/main/resources/tb-http-transport.yml | 7 +- .../src/main/resources/tb-mqtt-transport.yml | 7 +- 11 files changed, 219 insertions(+), 62 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e1296275cd..0cae3252ff 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -543,9 +543,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index b1afc61dbd..ba806c5daf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -38,6 +38,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -53,88 +54,99 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index edfcdc3d8e..eaffce241f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -47,71 +49,79 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 101f335536..3206f03250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -32,9 +32,11 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -46,59 +48,67 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index f15faa11da..2a5de01f86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -25,12 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -38,6 +34,11 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -50,33 +51,42 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producer); templateBuilder.responseTemplate(consumer); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -87,17 +97,17 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index cd575818f2..f3b6d3f10c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -19,32 +19,37 @@ import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.Duration; import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PushConfig; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @Slf4j public class TbPubSubAdmin implements TbQueueAdmin { + private static final String ACK_DEADLINE = "ackDeadlineInSec"; + private static final String MESSAGE_RETENTION = "messageRetentionInSec"; private final TbPubSubSettings pubSubSettings; private final SubscriptionAdminSettings subscriptionAdminSettings; private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + private final Map subscriptionProperties; - public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { + public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { this.pubSubSettings = pubSubSettings; + this.subscriptionProperties = subscriptionSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); @@ -149,8 +154,15 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - subscriptionAdminClient.createSubscription( - subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); + + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); + + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); subscriptionSet.add(subscriptionName.toString()); log.info("Created new subscription: [{}]", subscriptionName.toString()); } catch (IOException e) { @@ -159,4 +171,19 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } + private void setAckDeadline(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(ACK_DEADLINE)) { + builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); + } + } + + private void setMessageRetention(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { + Duration duration = Duration + .newBuilder() + .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) + .build(); + builder.setMessageRetentionDuration(duration); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index 851a2b3245..d6c6d0e271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -40,9 +40,6 @@ public class TbPubSubSettings { @Value("${queue.pubsub.service_account}") private String serviceAccount; - @Value("${queue.pubsub.ack_deadline}") - private int ackDeadline; - @Value("${queue.pubsub.max_msg_size}") private int maxMsgSize; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java new file mode 100644 index 0000000000..e98f51ec7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +public class TbPubSubSubscriptionSettings { + @Value("${queue.pubsub.queue-properties.core}") + private String coreProperties; + @Value("${queue.pubsub.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.pubsub.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.pubsub.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.pubsub.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreSettings; + @Getter + private Map ruleEngineSettings; + @Getter + private Map transportApiSettings; + @Getter + private Map notificationsSettings; + @Getter + private Map jsExecutorSettings; + + @PostConstruct + private void init() { + coreSettings = getSettings(coreProperties); + ruleEngineSettings = getSettings(ruleEngineProperties); + transportApiSettings = getSettings(transportApiProperties); + notificationsSettings = getSettings(notificationsProperties); + jsExecutorSettings = getSettings(jsExecutorProperties); + } + + private Map getSettings(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 78dc191e2c..c3e646c77c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -72,9 +72,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 82b548a456..0d48bda2b9 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -73,9 +73,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e3faf9cb87..4249757db1 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -103,9 +103,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" From 2ad4ddf1fbe0c1350e31a70e369fe96c0085bb84 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 21:05:12 +0300 Subject: [PATCH 10/39] pubsub improvements --- .../provider/PubSubMonolithQueueFactory.java | 21 ++++ .../provider/PubSubTbCoreQueueFactory.java | 18 ++++ .../PubSubTbRuleEngineQueueFactory.java | 18 ++++ .../provider/PubSubTransportQueueFactory.java | 18 ++++ .../server/queue/pubsub/TbPubSubAdmin.java | 101 +++++++++--------- 5 files changed, 124 insertions(+), 52 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index ba806c5daf..af0b276c1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -45,6 +45,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -153,4 +155,23 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index eaffce241f..0a23d58e46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -42,6 +42,8 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -128,4 +130,20 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 3206f03250..79501130ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -41,6 +41,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -117,4 +119,20 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 2a5de01f86..7e859e02ab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -111,4 +113,20 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index f3b6d3f10c..1241370d05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -40,9 +40,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { private static final String ACK_DEADLINE = "ackDeadlineInSec"; private static final String MESSAGE_RETENTION = "messageRetentionInSec"; + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + private final TbPubSubSettings pubSubSettings; - private final SubscriptionAdminSettings subscriptionAdminSettings; - private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); private final Map subscriptionProperties; @@ -51,6 +52,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { this.pubSubSettings = pubSubSettings; this.subscriptionProperties = subscriptionSettings; + TopicAdminSettings topicAdminSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -58,6 +60,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create TopicAdminSettings."); } + SubscriptionAdminSettings subscriptionAdminSettings; try { subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -65,7 +68,9 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create SubscriptionAdminSettings."); } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + try { + topicAdminClient = TopicAdminClient.create(topicAdminSettings); + ListTopicsRequest listTopicsRequest = ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); @@ -77,7 +82,8 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to get topics.", e); } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + try { + subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); ListSubscriptionsRequest listSubscriptionsRequest = ListSubscriptionsRequest.newBuilder() @@ -104,31 +110,21 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; } - - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); - createSubscriptionIfNotExists(partition, topicName); - } catch (IOException e) { - log.error("Failed to create topic: [{}].", topicName.toString(), e); - throw new RuntimeException("Failed to create topic.", e); } - } - - @Override - public void destroy() { + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); } private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { @@ -139,36 +135,27 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; } + } - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } catch (IOException e) { - log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); - throw new RuntimeException("Failed to create subscription.", e); - } + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); } private void setAckDeadline(Subscription.Builder builder) { @@ -186,4 +173,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { builder.setMessageRetentionDuration(duration); } } + + @Override + public void destroy() { + if (topicAdminClient != null) { + topicAdminClient.close(); + } + if (subscriptionAdminClient != null) { + subscriptionAdminClient.close(); + } + } } From c3bd2a0ebe2e7c005cae7520cc1ab6bf4e7224f8 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Sat, 18 Apr 2020 00:33:24 +0300 Subject: [PATCH 11/39] Fixed incorrect params in the get timeseries request --- .../thingsboard/rest/client/RestClient.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7e0dbf3858..f6a6541de0 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1646,7 +1646,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1998,17 +1998,25 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink) { - String urlParams = "limit={limit}&ascOrder={ascOrder}"; + return getUrlParams(pageLink, "startTime", "endTime"); + } + + private String getUrlParamsTs(TimePageLink pageLink) { + return getUrlParams(pageLink, "startTs", "endTs"); + } + + private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { + StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); if (pageLink.getStartTime() != null) { - urlParams += "&startTime={startTime}"; + urlParams.append("&").append(startTime).append("={startTime}"); } if (pageLink.getEndTime() != null) { - urlParams += "&endTime={endTime}"; + urlParams.append("&").append(endTime).append("={endTime}"); } if (pageLink.getIdOffset() != null) { - urlParams += "&offset={offset}"; + urlParams.append("&offset={offset}"); } - return urlParams; + return urlParams.toString(); } private String getUrlParams(TextPageLink pageLink) { From eab6ba459824a9916ab1438418fcc6447c119bcd Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Mon, 20 Apr 2020 19:38:59 +0300 Subject: [PATCH 12/39] StringBuilder replaced with String --- .../java/org/thingsboard/rest/client/RestClient.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index f6a6541de0..652de133b7 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2006,17 +2006,17 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { - StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); + String urlParams = "limit={limit}&ascOrder={ascOrder}"; if (pageLink.getStartTime() != null) { - urlParams.append("&").append(startTime).append("={startTime}"); + urlParams += "&" + startTime + "={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams.append("&").append(endTime).append("={endTime}"); + urlParams += "&" + endTime + "={endTime}"; } if (pageLink.getIdOffset() != null) { - urlParams.append("&offset={offset}"); + urlParams += "&offset={offset}"; } - return urlParams.toString(); + return urlParams; } private String getUrlParams(TextPageLink pageLink) { From 283ad27cb54d476e011b92e4de94aac465a89c39 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Apr 2020 17:08:51 +0300 Subject: [PATCH 13/39] added rabbitmq queue arguments --- .../src/main/resources/thingsboard.yml | 6 ++ .../provider/AwsSqsTransportQueueFactory.java | 11 ++- .../InMemoryTbTransportQueueFactory.java | 9 +- .../RabbitMqMonolithQueueFactory.java | 61 +++++++++--- .../provider/RabbitMqTbCoreQueueFactory.java | 57 ++++++++--- .../RabbitMqTbRuleEngineQueueFactory.java | 48 +++++++-- .../RabbitMqTransportQueueFactory.java | 50 ++++++++-- .../ServiceBusTransportQueueFactory.java | 6 +- .../queue/rabbitmq/TbRabbitMqAdmin.java | 11 +-- .../rabbitmq/TbRabbitMqQueueArguments.java | 98 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 6 ++ .../src/main/resources/tb-http-transport.yml | 6 ++ .../src/main/resources/tb-mqtt-transport.yml | 6 ++ 13 files changed, 319 insertions(+), 56 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cae3252ff..0b2c40acef 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index c9ba3ec15f..351cd4058c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -30,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -47,8 +48,10 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; @@ -56,12 +59,15 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @@ -94,7 +100,7 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override @@ -105,6 +111,9 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @PreDestroy private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } if (transportApiAdmin != null) { transportApiAdmin.destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index c03f5e52d8..2855e577ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -32,9 +32,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -43,13 +43,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -86,7 +89,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index ff4a69e2e6..72f45e5276 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -28,8 +28,10 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -37,6 +39,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -48,7 +52,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -56,7 +65,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,73 +73,97 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 5708e0738c..d84bbedbd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -34,13 +34,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -51,7 +55,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, TbQueueCoreSettings coreSettings, @@ -59,67 +68,91 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index e2755938d9..a18f427fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -48,55 +52,63 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @@ -105,4 +117,20 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index dc6154255c..841e004b3f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -30,12 +30,17 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -43,34 +48,45 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbQueueCoreSettings coreSettings, + TbRabbitMqQueueArguments queueArguments) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbRabbitMqProducerTemplate> producerTemplate = - new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); TbRabbitMqConsumerTemplate> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,17 +97,33 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index d38a4389a3..33c5277ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -29,6 +29,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -41,17 +42,20 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory private final TbServiceBusSettings serviceBusSettings; private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbQueueAdmin admin) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -82,7 +86,7 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 9a7e5db342..3bef6deb84 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,24 +18,23 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.concurrent.TimeoutException; @Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") public class TbRabbitMqAdmin implements TbQueueAdmin { private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; + private final Map arguments; - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { this.rabbitMqSettings = rabbitMqSettings; + this.arguments = arguments; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); @@ -55,7 +54,7 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String topic) { try { - channel.queueDeclare(topic, false, false, false, null); + channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { log.error("Failed to bind queue: [{}]", topic, e); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java new file mode 100644 index 0000000000..eb73e7dce6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqQueueArguments { + @Value("${queue.rabbitmq.queue-properties.core}") + private String coreProperties; + @Value("${queue.rabbitmq.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.rabbitmq.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.rabbitmq.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.rabbitmq.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreArgs; + @Getter + private Map ruleEngineArgs; + @Getter + private Map transportApiArgs; + @Getter + private Map notificationsArgs; + @Getter + private Map jsExecutorArgs; + + @PostConstruct + private void init() { + coreArgs = getArgs(coreProperties); + ruleEngineArgs = getArgs(ruleEngineProperties); + transportApiArgs = getArgs(transportApiProperties); + notificationsArgs = getArgs(notificationsProperties); + jsExecutorArgs = getArgs(jsExecutorProperties); + } + + private Map getArgs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } + return configs; + } + + private Object getObjectValue(String str) { + if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { + return Boolean.valueOf(str); + } else if (isNumeric(str)) { + return getNumericValue(str); + } + return str; + } + + private Object getNumericValue(String str) { + if (str.contains(".")) { + return Double.valueOf(str); + } else { + return Long.valueOf(str); + } + } + + private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); + + public boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + return PATTERN.matcher(strNum).matches(); + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3e646c77c..c3dcb76508 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -95,6 +95,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0d48bda2b9..baac6e977f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -96,6 +96,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 4249757db1..e920891d66 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -126,6 +126,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 53eb09bcb939ae28cbefefbbab5cb2dfe3d05434 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:31:25 +0300 Subject: [PATCH 14/39] Improvements to tests due to new architecture --- .../RuleChainActorMessageProcessor.java | 5 ++- .../server/actors/tenant/TenantActor.java | 40 ++++++++++++------- .../controller/TelemetryController.java | 8 ++-- .../queue/DefaultTbClusterService.java | 6 +++ .../DefaultTbRuleEngineConsumerService.java | 2 +- ...TbRuleEngineProcessingStrategyFactory.java | 16 ++++++-- .../rpc/DefaultTbCoreDeviceRpcService.java | 10 ++--- .../src/main/resources/thingsboard.yml | 2 +- ...tractMqttServerSideRpcIntegrationTest.java | 15 ++++--- .../AbstractMqttTelemetryIntegrationTest.java | 4 +- ...AbstractRuleEngineFlowIntegrationTest.java | 4 +- application/src/test/resources/logback.xml | 2 +- .../server/queue/memory/InMemoryStorage.java | 4 +- .../queue/memory/InMemoryTbQueueConsumer.java | 2 +- .../TbRuleEngineQueueConfiguration.java | 2 +- dao/src/test/resources/sql-test.properties | 10 ++++- 16 files changed, 87 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index a374e590b5..4e3d69c2b7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -249,7 +249,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - if (isRuleEngineForCurrentTenant) { - try { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; } - } catch (Exception e) { - cantFindTenant = true; } + log.info("[{}] Tenant actor started.", tenantId); } - log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -104,7 +111,12 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (cantFindTenant) { - log.info("Missing Tenant msg: {}", msg); + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; } switch (msg.getMsgType()) { case PARTITION_CHANGE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 8352e3f457..aee56652c4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -344,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -354,13 +354,13 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); @@ -397,7 +397,7 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( user.getTenantId(), deviceId, scope, attributes), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f0054771d9..926b08f38b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -87,6 +87,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); byte[] msgBytes = encodingService.encode(msg); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); @@ -96,6 +97,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -108,6 +110,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); toRuleEngineMsgs.incrementAndGet(); } @@ -123,6 +126,7 @@ public class DefaultTbClusterService implements TbClusterService { } } TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -134,6 +138,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -147,6 +152,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); toTransportNfs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index c8f126b26c..6736518bcc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -174,7 +174,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< })); boolean timeout = false; - if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { timeout = true; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index b6579b8dcb..bbf283e962 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; @@ -32,7 +34,7 @@ public class TbRuleEngineProcessingStrategyFactory { public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { switch (configuration.getType()) { - case "SKIP_ALL": + case "SKIP_ALL_FAILURES": return new SkipStrategy(name); case "RETRY_ALL": return new RetryStrategy(name, true, true, true, configuration); @@ -98,7 +100,7 @@ public class TbRuleEngineProcessingStrategyFactory { } log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } if (pauseBetweenRetries > 0) { try { @@ -123,7 +125,15 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + if (!result.isSuccess()) { + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index 8a1423f97c..f6e5eb8b43 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -106,7 +106,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { @Override public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -177,9 +177,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to rule engine request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for processing to rule engine request.", requestId); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); @@ -189,9 +189,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to device request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for to device request.", requestId); localToDeviceRpcRequests.remove(requestId); }, timeout, TimeUnit.MILLISECONDS); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0b2c40acef..3df3cb9559 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -622,7 +622,7 @@ queue: # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 485d618a01..9ddb2c81d9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; + private static final AtomicInteger atomicInteger = new AtomicInteger(2); + @Before public void beforeTest() throws Exception { @@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -130,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -139,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -169,7 +172,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC Thread.sleep(2000); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -187,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -196,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 7e329d8b05..f96f207834 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -107,13 +107,13 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); - client.connect(options).waitForCompletion(3000); + client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport - Thread.sleep(1000); + Thread.sleep(5000); doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 3343d271fe..87a181deae 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -152,7 +152,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -265,7 +265,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..7943b9d9b1 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index e119d1433b..f51f5b8ae0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -50,10 +50,10 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) throws InterruptedException { + public List get(String topic) throws InterruptedException { if (storage.containsKey(topic)) { List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + T first = (T) storage.get(topic).poll(); if (first != null) { entities = new ArrayList<>(); entities.add(first); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 3920f143f1..a619eda132 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -68,7 +68,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon .stream() .map(tpi -> { try { - return storage.get(tpi.getFullTopicName(), durationInMillis); + return storage.get(tpi.getFullTopicName()); } catch (InterruptedException e) { if (!stopped) { log.error("Queue was interrupted.", e); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index c5a24f20c6..019a76953b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -24,7 +24,7 @@ public class TbRuleEngineQueueConfiguration { private String topic; private int pollInterval; private int partitions; - private String packProcessingTimeout; + private long packProcessingTimeout; private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 13c0fbc818..7b5f82da2a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -35,4 +35,12 @@ service.type=monolith #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 + +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file From b2bb39f7a7c65c7e1ce8ee90ad17dcc6857ccf6e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:47:06 +0300 Subject: [PATCH 15/39] Restored WARN log level for tests --- application/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 7943b9d9b1..47dacce343 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + From 076cf655f87a914d1b882d35647faedaa8bd7c2e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 23:54:26 +0300 Subject: [PATCH 16/39] Improved tests --- .../thingsboard/server/controller/TelemetryController.java | 5 ----- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- dao/src/test/resources/sql-test.properties | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index aee56652c4..bb866e3e9b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -397,11 +397,6 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { - DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes), null); - } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f96f207834..0fe85fbb88 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -109,7 +109,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr client.setCallback(callback); client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"value\"}"; + String payload = "{\"key\":\"uniqueValue\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 7b5f82da2a..50d3a3feb3 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -37,6 +37,9 @@ service.type=monolith #spring.datasource.driverClassName=org.postgresql.Driver #spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 + queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main queue.rule-engine.queues[0].poll-interval=25 From dc3d1dd794f1c70cbd6a6eecc9a14aaf7abe45e4 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 13:03:53 +0300 Subject: [PATCH 17/39] added ttl for events --- .../main/data/upgrade/2.4.3/schema_update_ttl.sql | 15 +++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 5 +++++ .../service/ttl/events/EventsCleanUpService.java | 5 ++--- .../server/dao/model/ModelConstants.java | 3 +++ .../server/dao/model/sql/EventEntity.java | 12 ++++++++++++ dao/src/main/resources/sql/schema-entities.sql | 15 +++++++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index ff1fb5129b..bd40434ffe 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,3 +123,18 @@ BEGIN END LOOP; END $$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ + DECLARE + ttl_ts bigint; +BEGIN + IF system_ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; + diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..d7454db0b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index bb7d112bb1..e10ea30893 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - log.info("ttl: [{}]", ttl); - log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); - // TODO: 4/15/20 Do a clean up. + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..4a2f8aef23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,6 +32,9 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units + public static final long EPOCH_DIFF = 122192928000000000L; + /** * Generic constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..c0ad279d5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -37,6 +37,9 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; @@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data @EqualsAndHashCode(callSuper = true) @@ -73,9 +77,13 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; From 0103da1ba4852f305606c7eb52c4ffd560f78dbc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:12:57 +0300 Subject: [PATCH 18/39] Improved Callback and Destroy of ndoes --- .../queue/DefaultTbCoreConsumerService.java | 15 +++- .../DefaultTbRuleEngineConsumerService.java | 8 +- .../service/queue/TbMsgPackCallback.java | 8 +- ...t.java => TbMsgPackProcessingContext.java} | 4 +- .../server/service/queue/TbPackCallback.java | 28 ++---- .../queue/TbPackProcessingContext.java | 90 +++++++++++++++++++ .../processing/AbstractConsumerService.java | 11 ++- .../TbRuleEngineProcessingResult.java | 6 +- ...va => TbMsgPackProcessingContextTest.java} | 4 +- 9 files changed, 124 insertions(+), 50 deletions(-) rename application/src/main/java/org/thingsboard/server/service/queue/{ProcessingAttemptContext.java => TbMsgPackProcessingContext.java} (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java rename application/src/test/java/org/thingsboard/server/service/queue/{ProcessingAttemptContextTest.java => TbMsgPackProcessingContextTest.java} (94%) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c943e4c4d0..a5abd77ad5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -98,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToSubscriptionMgrMsg()) { @@ -147,8 +154,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } mainConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 6736518bcc..0c48c54412 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -64,9 +63,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -116,10 +112,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @PreDestroy public void stop() { + super.destroy(); if (submitExecutor != null) { submitExecutor.shutdownNow(); } - ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); } @@ -156,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< submitStrategy.init(msgs); while (!stopped) { - ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java index 2a6b6a658d..f093cc885a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -18,21 +18,17 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeException; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbMsgPackCallback implements TbMsgCallback { private final UUID id; private final TenantId tenantId; - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { + public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { this.id = id; this.tenantId = tenantId; this.ctx = ctx; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java index aefb1697cd..7e78ac6f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java @@ -29,7 +29,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -public class ProcessingAttemptContext { +public class TbMsgPackProcessingContext { private final TbRuleEngineSubmitStrategy submitStrategy; @@ -44,7 +44,7 @@ public class ProcessingAttemptContext { @Getter private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); this.pendingCount = new AtomicInteger(pendingMap.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index ba5a883ea0..5a5c172ee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -19,44 +19,26 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbPackCallback implements TbCallback { - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap failedMap; + private final TbPackProcessingContext ctx; private final UUID id; - public TbPackCallback(UUID id, - CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap failedMap) { + public TbPackCallback(UUID id, TbPackProcessingContext ctx) { this.id = id; - this.processingTimeoutLatch = processingTimeoutLatch; - this.ackMap = ackMap; - this.failedMap = failedMap; + this.ctx = ctx; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - T msg = ackMap.remove(id); - if (msg != null && ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onSuccess(id); } @Override public void onFailure(Throwable t) { log.trace("[{}] ON FAILURE", id, t); - T msg = ackMap.remove(id); - if (msg != null) { - failedMap.put(id, msg); - } - if (ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onFailure(id, t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java new file mode 100644 index 0000000000..e9f1224625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbPackProcessingContext { + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + + public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.processingTimeoutLatch = processingTimeoutLatch; + this.pendingCount = new AtomicInteger(ackMap.size()); + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + } + if (empty) { + processingTimeoutLatch.countDown(); + } else { + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T t : ackMap.values()) { + log.trace("left item: {}", t); + } + } + } + } + + public void onFailure(UUID id, Throwable t) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T v : ackMap.values()) { + log.trace("left item: {}", v); + } + } + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public ConcurrentMap getAckMap() { + return ackMap; + } + + public ConcurrentMap getFailedMap() { + return failedMap; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index a238e34337..c2705fcbdc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,11 +23,13 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.TbPackCallback; +import org.thingsboard.server.service.queue.TbPackProcessingContext; import javax.annotation.PreDestroy; import java.util.List; @@ -92,11 +94,12 @@ public abstract class AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { handleNotification(id, msg, callback); } catch (Throwable e) { @@ -105,8 +108,8 @@ public abstract class AbstractConsumerService log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); } nfConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index 8e0fcaa74a..e818ffb9d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.ProcessingAttemptContext; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import java.util.UUID; import java.util.concurrent.ConcurrentMap; @@ -32,9 +32,9 @@ public class TbRuleEngineProcessingResult { @Getter private final boolean timeout; @Getter - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { + public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { this.timeout = timeout; this.ctx = ctx; this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java similarity index 94% rename from application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java rename to application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java index 2de2414f71..43cd62ea97 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -37,7 +37,7 @@ import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) -public class ProcessingAttemptContextTest { +public class TbMsgPackProcessingContextTest { @Test public void testHighConcurrencyCase() throws InterruptedException { @@ -51,7 +51,7 @@ public class ProcessingAttemptContextTest { messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); } when(strategyMock.getPendingMap()).thenReturn(messages); - ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); for (UUID uuid : messages.keySet()) { for (int i = 0; i < parallelCount; i++) { executorService.submit(() -> context.onSuccess(uuid)); From 26a200dcfb1b610be6091b34edf7e76694ec8a3b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:27:16 +0300 Subject: [PATCH 19/39] Renaming the queues --- .../server/controller/QueueController.java | 2 +- .../TbRuleEngineSubmitStrategyFactory.java | 4 ++-- .../src/main/resources/thingsboard.yml | 19 +++++++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 8d1cec7f4b..a0b46f9166 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -43,7 +43,7 @@ public class QueueController extends BaseController { ServiceType type = ServiceType.valueOf(serviceType); switch (type) { case TB_RULE_ENGINE: - return Arrays.asList("HighPriority", "Main"); + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); default: return Collections.emptyList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java index f5a7457c17..90eb8561a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -29,9 +29,9 @@ public class TbRuleEngineSubmitStrategyFactory { return new BurstTbRuleEngineSubmitStrategy(name); case "BATCH": return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); - case "SEQUENTIAL_WITHIN_ORIGINATOR": + case "SEQUENTIAL_BY_ORIGINATOR": return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL_WITHIN_TENANT": + case "SEQUENTIAL_BY_TENANT": return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); case "SEQUENTIAL": return new SequentialTbRuleEngineSubmitStrategy(name); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3df3cb9559..9949a84c4d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -618,7 +618,7 @@ queue: partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: @@ -633,7 +633,7 @@ queue: partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch processing-strategy: @@ -642,6 +642,21 @@ queue: retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" From 71bf15caf79e7cd25f36227e969bf7910b3e0fc0 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:16:58 +0300 Subject: [PATCH 20/39] improvements --- .../data/upgrade/2.4.3/schema_update_ttl.sql | 24 ++++++++++++----- .../ttl/events/EventsCleanUpService.java | 7 +++-- .../src/main/resources/thingsboard.yml | 9 ++++--- .../resources/sql/schema-entities-hsql.sql | 27 +++++++++++++++++++ .../main/resources/sql/schema-entities.sql | 21 +++++++++++---- 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index bd40434ffe..dda3bd7b1c 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -124,17 +124,27 @@ BEGIN END $$; -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ - DECLARE - ttl_ts bigint; +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; - diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index e10ea30893..a608ca257b 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -31,9 +31,12 @@ import java.sql.SQLException; @Service public class EventsCleanUpService extends AbstractCleanUpService { - @Value("${sql.ttl.events.events_key_value_ttl}") + @Value("${sql.ttl.events.events_ttl}") private long ttl; + @Value("${sql.ttl.events.debug_events_ttl}") + private long debugTtl; + @Value("${sql.ttl.events.enabled}") private boolean ttlTaskExecutionEnabled; @@ -50,7 +53,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6eee7bc32a..4b584ab44f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,12 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: @@ -371,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index f28f7f5ebd..d4900020c6 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -251,3 +252,29 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a70eb13ab7..931856e1df 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint, + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -253,16 +253,27 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ DECLARE ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; From a0b4a46c3da516c1405932128ac87e3f770b7797 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:20:23 +0300 Subject: [PATCH 21/39] fix typo --- application/src/main/resources/thingsboard.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4b584ab44f..cb343e8bdb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,13 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: From 005d70d2eaf4051a28168c651d809574c9c98f6c Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:22:10 +0300 Subject: [PATCH 22/39] fix yml typo --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cb343e8bdb..456761a0cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -372,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: From ce086f915ea272504ce756c8e1c018a772c7a28b Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:53:38 +0300 Subject: [PATCH 23/39] try to pass travis --- .../thingsboard/server/install/ThingsboardInstallService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 842550d209..8dd45bc41b 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; From 6d20ed05501f4b2f42a5922f6f79a5f7b2af95d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 18:09:47 +0300 Subject: [PATCH 24/39] fix event insert repository --- .../event/AbstractEventInsertRepository.java | 3 ++- .../sql/event/HsqlEventInsertRepository.java | 4 +-- .../sql/event/PsqlEventInsertRepository.java | 2 +- .../resources/sql/schema-entities-hsql.sql | 25 ------------------- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- 6 files changed, 7 insertions(+), 31 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index b8d3af0968..8a341e0f81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -75,7 +75,8 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()); + .setParameter("tenant_id", entity.getTenantId()) + .setParameter("ts", entity.getTs()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index bebfd7c5e0..f5a00fe7f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -44,7 +44,7 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java index 1a8c12cb45..937262241a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java @@ -48,6 +48,6 @@ public class PsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) { - return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, " + updateKeyStatement + " returning *"; + return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, ts = :ts," + updateKeyStatement + " returning *"; } } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index d4900020c6..697e34930a 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -253,28 +253,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; - diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 32e2a78620..bb0a964c13 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3789444106..f83e80931b 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS From 5dbf7d764cceabb160fc168b579d4819c09e5b71 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 18:52:34 +0300 Subject: [PATCH 25/39] Default Dashboards --- .../data/json/demo/dashboards/gateways.json | 1274 +++++++++++++++++ .../dashboards/rule_engine_statistics.json | 520 +++++++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../DefaultTelemetryWebSocketService.java | 8 +- .../src/main/resources/thingsboard.yml | 15 +- .../TbSynchronizationBeginNode.java | 2 +- .../transaction/TbSynchronizationEndNode.java | 2 +- 7 files changed, 1811 insertions(+), 12 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/gateways.json create mode 100644 application/src/main/data/json/demo/dashboards/rule_engine_statistics.json diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json new file mode 100644 index 0000000000..f6370696c1 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -0,0 +1,1274 @@ +{ + "title": "Gateways", + "configuration": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "isSystemType": true, + "bundleAlias": "entity_admin_widgets", + "typeAlias": "device_admin_table2", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "entitiesTitle": "List of gateways", + "enableSelectColumnDisplay": true, + "displayEntityLabel": false, + "entityNameColumnTitle": "Gateway Name" + }, + "title": "Devices gateway table", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.3646047595211721 + }, + { + "name": "eventsSent", + "type": "timeseries", + "label": "Sent", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.7235710720767985 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Events", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5085933386303254 + }, + { + "name": "LOGS", + "type": "timeseries", + "label": "Latest log", + "color": "#ffc107", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3504240371585048, + "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" + }, + { + "name": "RemoteLoggingLevel", + "type": "attribute", + "label": "Log level", + "color": "#607d8b", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9785994222542516 + } + ], + "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" + } + ], + "showTitleIcon": true, + "titleIcon": "list", + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "List device", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", + "name": "Add device", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n\n", + "customCss": "", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "78845501-234e-a452-6819-82b5b776e99f", + "name": "Configuration", + "icon": "settings", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "openRightLayout": false, + "setEntityId": true + }, + { + "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", + "name": "Graphs", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname_grafic", + "setEntityId": true + }, + { + "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", + "name": "Edit device", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + }, + { + "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", + "name": "Delete device", + "icon": "delete", + "type": "custom", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" + } + ], + "rowClick": [ + { + "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", + "name": "to_entityname__config", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "setEntityId": true, + "stateEntityParamName": "" + } + ] + } + }, + "id": "94715984-ae74-76e4-20b7-2f956b01ed80" + }, + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 14, + "sizeY": 13, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "LOGS", + "type": "timeseries", + "label": "LOGS", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3496649158709739, + "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 2592000000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Debug events (logs)", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" + }, + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "opcuaEventsProduced", + "type": "timeseries", + "label": "opcuaEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.1477920581839779 + }, + { + "name": "opcuaEventsSent", + "type": "timeseries", + "label": "opcuaEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6500957113784758 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 7, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "Events", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.8156044798125357 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Produced", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.6538259344015449 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 604800000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 6, + "hideEmptyLines": true + }, + "title": "Total Messages", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": false, + "showMax": false, + "showAvg": true, + "showTotal": false + } + }, + "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "eventsSent", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.41414001784591314 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "eventsProduced", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.7819101846284422 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "History information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "bleEventsProduced", + "type": "timeseries", + "label": "bleEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.5625165504526104 + }, + { + "name": "bleEventsSent", + "type": "timeseries", + "label": "bleEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6817950080745288 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 5000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "attributes_card", + "type": "latest", + "title": "New widget", + "sizeX": 7, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "eventsTitle": "Gateway Events Form", + "eventsReg": [ + "EventsProduced", + "EventsSent" + ] + }, + "title": "Gateway events", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "466f046d-6005-a168-b107-60fcb2469cd5" + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "config_form_latest", + "type": "latest", + "title": "New widget", + "sizeX": 10, + "sizeY": 9, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "gatewayTitle": "Gateway configuration (Single device)", + "readOnly": false + }, + "title": "New Gateway configuration (Single device)", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodParams": "{}", + "methodName": "gateway_reboot", + "buttonText": "GATEWAY REBOOT" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodName": "gateway_restart", + "methodParams": "{}", + "buttonText": "gateway restart" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.9743324774725604 + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.5530093635101525 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [], + "useMarkerImageFunction": false, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.Mapnik", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5 + }, + "title": "Gateway Location", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" + } + }, + "states": { + "main_gateway": { + "name": "Gateways", + "root": true, + "layouts": { + "main": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname__config": { + "name": "${entityName} Configuration", + "root": false, + "layouts": { + "main": { + "widgets": { + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "sizeX": 14, + "sizeY": 13, + "row": 0, + "col": 10 + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "sizeX": 10, + "sizeY": 9, + "row": 0, + "col": 0 + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "sizeX": 4, + "sizeY": 2, + "row": 9, + "col": 0 + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "sizeX": 4, + "sizeY": 2, + "row": 11, + "col": 0 + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "sizeX": 6, + "sizeY": 4, + "row": 9, + "col": 4 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname_grafic": { + "name": "${entityName} Details", + "root": false, + "layouts": { + "main": { + "widgets": { + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 4, + "col": 7 + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "sizeX": 7, + "sizeY": 7, + "row": 5, + "col": 0 + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 0, + "col": 7 + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 8, + "col": 7 + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "sizeX": 7, + "sizeY": 5, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "auto 100%", + "autoFillHeight": true, + "mobileAutoFillHeight": true, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "3e0f533a-0db1-3292-184f-06e73535061a": { + "id": "3e0f533a-0db1-3292-184f-06e73535061a", + "alias": "Gateways", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "gateway", + "deviceNameFilter": "" + } + }, + "b2487e75-2fa4-f211-142c-434dfd50c70c": { + "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", + "alias": "Current Gateway", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "settings": { + "stateControllerId": "entity", + "showTitle": true, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true, + "titleColor": "rgba(0,0,0,0.870588)" + } + }, + "name": "Gateways" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json new file mode 100644 index 0000000000..b8231ab880 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -0,0 +1,520 @@ +{ + "title": "Rule Engine Statistics", + "configuration": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "successfulMsgs", + "type": "timeseries", + "label": "${entityName} Successful", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.15490750967648736 + }, + { + "name": "failedMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Failures", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.4186621166514697 + }, + { + "name": "tmpFailed", + "type": "timeseries", + "label": "${entityName} Processing Failures", + "color": "#ffc107", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.49891007198715376 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Queue Stats", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 24, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Chain", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleChainName;" + }, + "_hash": 0.9954481282345906 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Node", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleNodeName;" + }, + "_hash": 0.18580357036589978 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Latest Error", + "color": "#f44336", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).message;" + }, + "_hash": 0.7255162989552142 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Exceptions", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "timeoutMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Timeouts", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.565222981550328 + }, + { + "name": "tmpTimeout", + "type": "timeseries", + "label": "${entityName} Processing Timeouts", + "color": "#9c27b0", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.2679547062508352 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Processing Failures and Timeouts", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" + } + }, + "states": { + "default": { + "name": "Rule Engine Statistics", + "root": true, + "layouts": { + "main": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 0 + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "sizeX": 24, + "sizeY": 5, + "row": 7, + "col": 0 + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 12 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "alias": "TbServiceQueues", + "filter": { + "type": "assetType", + "resolveMultiple": true, + "assetType": "TbServiceQueue", + "assetNameFilter": "" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1586176634823, + "endTimeMs": 1586263034823 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Rule Engine Statistics" +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 78319997a9..c362c5e889 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -79,7 +79,7 @@ public class TbCoreConsumerStats { public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + " deviceState [{}] subMgr [{}] coreNfs [{}]", total, sessionEventCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 5e9e83b8aa..6afcaf97bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -345,6 +345,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -353,7 +355,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(false) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } @@ -440,6 +442,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -448,7 +452,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(true) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9949a84c4d..6a16349c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -410,8 +410,9 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: @@ -589,7 +590,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic @@ -615,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -630,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -643,9 +644,9 @@ queue: failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 31501688eb..751b061666 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -33,7 +33,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Starts synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index ac3d42ef36..8b83bff1fa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,7 +35,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Stops synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") From 087ea2d691b900d4f8cafc9dd055f7cad7019137 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Apr 2020 09:25:25 +0300 Subject: [PATCH 26/39] Commented Unstable Test --- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 0fe85fbb88..47e9537ef9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -97,7 +97,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - @Test + +// @Test - Unstable public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); From 9a052ef0bc333c8864b6ac9bc228d6290e0629d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Thu, 23 Apr 2020 12:39:33 +0300 Subject: [PATCH 27/39] fix typo in EventEntity class --- .../java/org/thingsboard/server/dao/model/sql/EventEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index 0a541e24b0..3790577144 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -113,6 +113,6 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity Date: Thu, 23 Apr 2020 14:10:45 +0300 Subject: [PATCH 28/39] Changed default partition sizes --- application/src/main/resources/thingsboard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6a16349c4c..7457cc8c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -616,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -631,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -646,7 +646,7 @@ queue: - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL From 1510fe8448002e5c0f73a7ed957a0d52721df8dc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:22:34 +0300 Subject: [PATCH 29/39] Default Dashboard and Rule Chain --- .../json/demo/dashboards/theromstats.json | 1236 +++++++++++++++++ .../demo/rule_chains/root_rule_chain.json | 146 ++ .../demo/rule_chains/thermostat_alarms.json | 137 ++ .../DefaultSystemDataLoaderService.java | 36 +- .../service/install/InstallScripts.java | 53 +- 5 files changed, 1604 insertions(+), 4 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/theromstats.json create mode 100644 application/src/main/data/json/demo/rule_chains/root_rule_chain.json create mode 100644 application/src/main/data/json/demo/rule_chains/thermostat_alarms.json diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/theromstats.json new file mode 100644 index 0000000000..bd191a6bc5 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/theromstats.json @@ -0,0 +1,1236 @@ +{ + "title": "Thermostats", + "configuration": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "entities_table", + "type": "latest", + "title": "New widget", + "sizeX": 11, + "sizeY": 11, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "enableSelectColumnDisplay": false, + "entitiesTitle": "Thermostats", + "displayEntityLabel": false, + "entityNameColumnTitle": "Thermostat name" + }, + "title": "Thermostats", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value === \"true\") {\n color = 'rgb(39, 134, 34)';\n} else {\n color = 'rgb(255, 0, 0)';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.9264526512320641 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9801965063904188, + "units": "°C", + "decimals": 1 + }, + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5726727868178358, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "85b803db-90f2-5c63-1388-a378e0eb10d6", + "name": "Edit location", + "icon": "map", + "type": "openDashboardState", + "targetDashboardStateId": "map", + "setEntityId": false + }, + { + "id": "8ab5a518-67d2-b6a2-956d-81fd512294b2", + "name": "Add", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add thermostat

\n \n \n \n \n
\n \n \n
\n \n \n \n
\n
Thermostat name is required.
\n
\n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id);\n updateAliasData();\n $mdDialog.hide();\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + } + ], + "actionCellButton": [ + { + "id": "ca241cd8-788d-5508-a9ce-74b03ef42a7f", + "name": "Chart", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + }, + { + "id": "7506576f-87ba-d3a0-88fb-e304d451776d", + "name": "Edit", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit thermostat {{vm.entityName}}

\n \n \n \n \n
\n \n \n
\n \n \n \n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Save\n Cancel\n \n
\n", + "customCss": ".edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n attributeService = $injector.get('attributeService');\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditEntityDialogController($scope,$mdDialog) {\n var vm = this;\n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.entityType = entityId.entityType;\n vm.attributes = {};\n vm.serverAttributes = {};\n getEntityInfo();\n \n vm.save = function() {\n saveAttributes();\n $mdDialog.hide();\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n vm.serverAttributes = angular.copy(vm.attributes);\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n function(data){\n if (data.length) {\n getEntityAttributes(data);\n }\n });\n }\n \n function saveAttributes() {\n var attributesArray = [];\n for (var key in vm.attributes) {\n if (vm.attributes[key] !== vm.serverAttributes[key]) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n}" + }, + { + "id": "3488848b-e47d-6af6-659f-5d78369ece5e", + "name": "Delete", + "icon": "delete", + "type": "custom", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n deviceService = $injector.get('deviceService')\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteEntityDialog();\n\nfunction openDeleteEntityDialog() {\n var title = 'Delete thermostat \"' + entityName + '\"';\n var content = 'Are you sure you want to delete the thermostat \"' +\n entityName + '\"?';\n var confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(function() {\n deleteEntity();\n })\n}\n\nfunction deleteEntity() {\n deviceService.deleteDevice(entityId.id).then(\n function success() {\n updateAliasData();\n },\n function fail() {\n showErrorDialog();\n }\n );\n}\n\nfunction updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}\n\nfunction showErrorDialog() {\n var title = 'Error';\n var content = 'An error occurred while deleting the thermostat. Please try again.';\n var alert = $mdDialog.alert()\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .parent(angular.element($document[0].body))\n .targetEvent($event)\n .multiple(true)\n .clickOutsideToClose(true)\n .ok('CLOSE');\n $mdDialog.show(alert);\n}" + } + ], + "rowClick": [ + { + "id": "e3928f23-c135-0766-71d5-65ed61e0ce8d", + "name": "show alarm", + "icon": "more_horiz", + "type": "updateDashboardState", + "targetDashboardStateId": "default", + "setEntityId": true, + "stateEntityParamName": "alarm" + } + ] + } + }, + "id": "f33c746c-0dfc-c212-395b-b448c8a17209" + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "isSystemType": true, + "bundleAlias": "alarm_widgets", + "typeAlias": "alarms_table", + "type": "alarm", + "title": "New widget", + "sizeX": 13, + "sizeY": 5, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSelection": true, + "enableSearch": true, + "displayDetails": true, + "allowAcknowledgment": true, + "allowClear": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "-createdTime", + "enableSelectColumnDisplay": false, + "enableStatusFilter": true, + "alarmsTitle": "Alarms" + }, + "title": "New Alarms table", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "alarmSource": { + "type": "entity", + "dataKeys": [ + { + "name": "createdTime", + "type": "alarm", + "label": "Created time", + "color": "#2196f3", + "settings": {}, + "_hash": 0.7308410188824108 + }, + { + "name": "originator", + "type": "alarm", + "label": "Originator", + "color": "#4caf50", + "settings": {}, + "_hash": 0.056085530105439485 + }, + { + "name": "type", + "type": "alarm", + "label": "Type", + "color": "#f44336", + "settings": {}, + "_hash": 0.10212012352561795 + }, + { + "name": "severity", + "type": "alarm", + "label": "Severity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1777349980531262 + }, + { + "name": "status", + "type": "alarm", + "label": "Status", + "color": "#607d8b", + "settings": {}, + "_hash": 0.7977920750136249 + } + ], + "entityAliasId": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "name": "alarms" + }, + "alarmSearchStatus": "ANY", + "alarmsPollingInterval": 5, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": {}, + "datasources": [], + "alarmsMaxCountLoad": 0, + "alarmsFetchSize": 100 + }, + "id": "7943196b-eedb-d422-f9c3-b32d379ad172" + }, + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.7852346160709658, + "units": "°C", + "decimals": 1 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Temperature", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "14a19183-f0b2-d6be-0f62-9863f0a51111" + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.28640715926957183, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Humidity", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "07f49fd5-a73b-d74c-c220-362c20af81f4" + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "update_multiple_attributes", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "alarmTemperature", + "type": "attribute", + "label": "High temperature alarm", + "color": "#4caf50", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.8725278440159361 + }, + { + "name": "thresholdTemperature", + "type": "attribute", + "label": "High temperature threshold, °C", + "color": "#f44336", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmTemperature" + }, + "_hash": 0.7316078472857874 + }, + { + "name": "alarmHumidity", + "type": "attribute", + "label": "Low humidity alarm", + "color": "#ffc107", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.5339673667431057 + }, + { + "name": "thresholdHumidity", + "type": "attribute", + "label": "Low humidity threshold, %", + "color": "#607d8b", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmHumidity" + }, + "_hash": 0.2687091190358901 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showActionButtons": false, + "showResultMessage": true, + "fieldsAlignment": "column", + "fieldsInRow": 2, + "groupTitle": "${entityName}", + "widgetTitle": "Termostat settings" + }, + "title": "New Update Multiple Attributes", + "dropShadow": true, + "enableFullscreen": false, + "enableDataExport": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "c4631f94-2db3-523b-4d09-2a1a0a75d93f" + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "isSystemType": true, + "bundleAlias": "maps_v2", + "typeAlias": "openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 13, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#2196f3", + "settings": {}, + "_hash": 0.1371919646686739, + "decimals": 1, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#4caf50", + "settings": {}, + "_hash": 0.043177186765847475, + "decimals": 0, + "postFuncBody": "return value || \"\";" + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#f44336", + "settings": {}, + "_hash": 0.5548964320315584 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1803778014971602 + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.30926987994082844 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Thermostat details
", + "markerImageSize": 48, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "colorFunction": "\n", + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "defaultCenterPosition": [ + 0, + 0 + ], + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "useLabelFunction": true, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "defaultZoomLevel": 14, + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;" + }, + "title": "Thermostat maps", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "headerButton": [], + "tooltipAction": [ + { + "id": "bef25673-b37a-8821-bc0f-5d6dd3680f24", + "name": "navigate_to_details", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb" + }, + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "00fb2742-ba1f-7e43-673f-d6c08b72ed06" + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "0a430429-9078-9ae6-2b67-e4a15a2bf8bf" + } + }, + "states": { + "default": { + "name": "Thermostat", + "root": true, + "layouts": { + "main": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "sizeX": 11, + "sizeY": 11, + "row": 0, + "col": 0 + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "sizeX": 13, + "sizeY": 5, + "row": 0, + "col": 11 + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "sizeX": 13, + "sizeY": 6, + "row": 5, + "col": 11 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "map": { + "name": "Edit location", + "root": false, + "layouts": { + "main": { + "widgets": { + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "chart": { + "name": "${entityName}", + "root": false, + "layouts": { + "main": { + "widgets": { + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 0, + "col": 6 + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 6, + "col": 6 + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "sizeX": 6, + "sizeY": 6, + "row": 0, + "col": 0 + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "sizeX": 6, + "sizeY": 6, + "row": 6, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "68a058e1-fdda-8482-715b-3ae4a488568e": { + "id": "68a058e1-fdda-8482-715b-3ae4a488568e", + "alias": "Thermostats", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "thermostat", + "deviceNameFilter": "" + } + }, + "12ae98c7-1ea2-52cf-64d5-763e9d993547": { + "id": "12ae98c7-1ea2-52cf-64d5-763e9d993547", + "alias": "Thermostat", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": null, + "defaultStateEntity": null + } + }, + "ce27a9d0-93bf-b7a4-054d-d0369a8cf813": { + "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "alias": "Thermostat-alarm", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "alarm", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1587473857304, + "endTimeMs": 1587560257304 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Thermostats" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json new file mode 100644 index 0000000000..1b321b1e1e --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -0,0 +1,146 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Root Rule Chain", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 2, + "nodes": [ + { + "additionalInfo": { + "layoutX": 824, + "layoutY": 156 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": false, + "configuration": { + "defaultTTL": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 52 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", + "name": "Save Client Attributes", + "debugMode": false, + "configuration": { + "scope": "CLIENT_SCOPE" + } + }, + { + "additionalInfo": { + "layoutX": 347, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", + "name": "Message Type Switch", + "debugMode": false, + "configuration": { + "version": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 266 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log RPC from Device", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 379 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log Other", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 468 + }, + "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", + "name": "RPC Call Request", + "debugMode": false, + "configuration": { + "timeoutInSeconds": 60 + } + }, + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 90 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": false, + "configuration": { + "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" + } + } + ], + "connections": [ + { + "fromIndex": 0, + "toIndex": 6, + "type": "Success" + }, + { + "fromIndex": 2, + "toIndex": 4, + "type": "Other" + }, + { + "fromIndex": 2, + "toIndex": 1, + "type": "Post attributes" + }, + { + "fromIndex": 2, + "toIndex": 0, + "type": "Post telemetry" + }, + { + "fromIndex": 2, + "toIndex": 3, + "type": "RPC Request from Device" + }, + { + "fromIndex": 2, + "toIndex": 5, + "type": "RPC Request to Device" + } + ], + "ruleChainConnections": [ + { + "fromIndex": 6, + "targetRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "83d42540-85fd-11ea-aee2-794850541ced" + }, + "additionalInfo": { + "layoutX": 1088, + "layoutY": 203, + "ruleChainNodeId": "rule-chain-node-9" + }, + "type": "True" + } + ] + } +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json new file mode 100644 index 0000000000..9b7e3401d2 --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -0,0 +1,137 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Thermostat Alarms", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 5, + "nodes": [ + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 67 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;", + "severity": "MAJOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 201 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 131 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;", + "severity": "MINOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 275 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 586, + "layoutY": 148 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", + "name": "Check Alarms", + "debugMode": true, + "configuration": { + "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" + } + }, + { + "additionalInfo": { + "layoutX": 321, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", + "name": "Fetch Configuration", + "debugMode": true, + "configuration": { + "clientAttributeNames": [], + "sharedAttributeNames": [], + "serverAttributeNames": [ + "alarmTemperature", + "thresholdTemperature", + "alarmHumidity", + "thresholdHumidity" + ], + "latestTsKeyNames": [], + "tellFailureIfAbsent": false, + "getLatestValueWithTs": false + } + } + ], + "connections": [ + { + "fromIndex": 4, + "toIndex": 0, + "type": "NewTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 1, + "type": "ClearTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 2, + "type": "NewHumidityAlarm" + }, + { + "fromIndex": 4, + "toIndex": 3, + "type": "ClearHumidityAlarm" + }, + { + "fromIndex": 5, + "toIndex": 4, + "type": "Success" + } + ], + "ruleChainConnections": null + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd907714fa..f024a328c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,24 +25,32 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.Arrays; + @Service @Profile("install") @Slf4j @@ -76,6 +84,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + @Autowired private DeviceCredentialsService deviceCredentialsService; @@ -120,7 +131,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { demoTenant.setRegion("Global"); demoTenant.setTitle("Tenant"); demoTenant = tenantService.saveTenant(demoTenant); - installScripts.createDefaultRuleChains(demoTenant.getId()); + installScripts.loadDemoRuleChains(demoTenant.getId()); createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant"); Customer customerA = new Customer(); @@ -152,6 +163,25 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); + DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + + attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50)))); + + attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30)))); + installScripts.loadDashboards(demoTenant.getId(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 73e1dbf7fd..d61ca861eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,6 +24,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -182,4 +183,54 @@ public class InstallScripts { } + public void loadDemoRuleChains(TenantId tenantId) throws Exception { + Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR); + try { + JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + ruleChain.setTenantId(tenantId); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + + JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile()); + RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class); + + RuleChainId thermostatsRuleChainId = ruleChain.getId(); + rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId)); + rootChain.setTenantId(tenantId); + rootChain = ruleChainService.saveRuleChain(rootChain); + rootChainMetaData.setRuleChainId(rootChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json", e); + throw new RuntimeException("Unable to load dashboard from json", e); + } + + try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { + dirStream.forEach( + path -> { + try { + JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + + ruleChain.setTenantId(tenantId); + if (ruleChain.getName().equals("Root Rule Chain")) { + ruleChain.setRoot(true); + } + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json: [{}]", path.toString()); + throw new RuntimeException("Unable to load dashboard from json", e); + } + } + ); + } + } } From 9b0e92cac30159ea1bb1fc2eeb0fc978f4fa8d72 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:46:48 +0300 Subject: [PATCH 30/39] New Default Dashboard and Rule Chains --- .../server/service/install/DefaultSystemDataLoaderService.java | 2 +- .../org/thingsboard/server/service/install/InstallScripts.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index f024a328c7..b8d5d26d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index d61ca861eb..37f724ad22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From a91a69c57f6737fbdd438e5aeb79eb5bf47e9a13 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 13:55:06 +0300 Subject: [PATCH 31/39] Fixed Docker Compose --- .../service/install/InstallScripts.java | 24 ------------- .../service/script/RemoteJsInvokeService.java | 2 +- docker/docker-compose.yml | 21 ++++------- docker/kafka.env | 2 +- docker/tb-coap-transport.env | 1 + docker/tb-http-transport.env | 1 + docker/tb-mqtt-transport.env | 1 + docker/tb-node.env | 4 +-- docker/tb-node/conf/logback.xml | 4 +-- docker/tb-node/conf/thingsboard.conf | 2 +- docker/tb-transports/coap/conf/logback.xml | 4 +-- .../coap/conf/tb-coap-transport.conf | 2 +- docker/tb-transports/http/conf/logback.xml | 4 +-- .../http/conf/tb-http-transport.conf | 2 +- docker/tb-transports/mqtt/conf/logback.xml | 4 +-- .../mqtt/conf/tb-mqtt-transport.conf | 2 +- k8s/tb-coap-transport-configmap.yml | 6 ++-- k8s/tb-http-transport-configmap.yml | 6 ++-- k8s/tb-mqtt-transport-configmap.yml | 6 ++-- k8s/tb-node-configmap.yml | 6 ++-- k8s/thingsboard.yml | 36 +++++++------------ .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 24 files changed, 53 insertions(+), 93 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 37f724ad22..5181834b80 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -208,29 +208,5 @@ public class InstallScripts { log.error("Unable to load dashboard from json", e); throw new RuntimeException("Unable to load dashboard from json", e); } - - try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { - dirStream.forEach( - path -> { - try { - JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); - RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); - - ruleChain.setTenantId(tenantId); - if (ruleChain.getName().equals("Root Rule Chain")) { - ruleChain.setRoot(true); - } - ruleChain = ruleChainService.saveRuleChain(ruleChain); - - ruleChainMetaData.setRuleChainId(ruleChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); - } catch (Exception e) { - log.error("Unable to load dashboard from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load dashboard from json", e); - } - } - ); - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..fe2e7e8071 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -46,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; @Getter diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c36356b1cb..9774346b3d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -59,8 +59,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb1 - CLUSTER_NODE_ID: tb1 + TB_SERVICE_ID: tb1 env_file: - tb-node.env volumes: @@ -81,8 +80,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb2 - CLUSTER_NODE_ID: tb2 + TB_SERVICE_ID: tb2 env_file: - tb-node.env volumes: @@ -98,8 +96,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport1 - CLUSTER_NODE_ID: tb-mqtt-transport1 + TB_SERVICE_ID: tb-mqtt-transport1 env_file: - tb-mqtt-transport.env volumes: @@ -113,8 +110,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport2 - CLUSTER_NODE_ID: tb-mqtt-transport2 + TB_SERVICE_ID: tb-mqtt-transport2 env_file: - tb-mqtt-transport.env volumes: @@ -128,8 +124,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport1 - CLUSTER_NODE_ID: tb-http-transport1 + TB_SERVICE_ID: tb-http-transport1 env_file: - tb-http-transport.env volumes: @@ -143,8 +138,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport2 - CLUSTER_NODE_ID: tb-http-transport2 + TB_SERVICE_ID: tb-http-transport2 env_file: - tb-http-transport.env volumes: @@ -158,8 +152,7 @@ services: ports: - "5683:5683/udp" environment: - TB_HOST: tb-coap-transport - CLUSTER_NODE_ID: tb-coap-transport + TB_SERVICE_ID: tb-coap-transport env_file: - tb-coap-transport.env volumes: diff --git a/docker/kafka.env b/docker/kafka.env index 485b3c0169..81c01dae93 100644 --- a/docker/kafka.env +++ b/docker/kafka.env @@ -4,7 +4,7 @@ KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE -KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 +KAFKA_CREATE_TOPICS=js_eval.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 KAFKA_AUTO_CREATE_TOPICS_ENABLE=false KAFKA_LOG_RETENTION_BYTES=1073741824 KAFKA_LOG_SEGMENT_BYTES=268435456 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index ed8f78d116..af95bc904d 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -3,4 +3,5 @@ COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index c8363230a2..71bc434241 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -3,4 +3,5 @@ HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index b024c7a1a5..163cfab451 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -3,4 +3,5 @@ MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index 963943dccd..aec55e9205 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -2,9 +2,9 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 -RPC_HOST=${TB_HOST} +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=remote +JS_EVALUATOR=local TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index 4eb8a6a44f..bc694d704d 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-node/conf/thingsboard.conf b/docker/tb-node/conf/thingsboard.conf index 392b3302fd..3f27bf2859 100644 --- a/docker/tb-node/conf/thingsboard.conf +++ b/docker/tb-node/conf/thingsboard.conf @@ -15,7 +15,7 @@ # export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/coap/conf/logback.xml b/docker/tb-transports/coap/conf/logback.xml index f1ad75fd5e..20d4b6bc92 100644 --- a/docker/tb-transports/coap/conf/logback.xml +++ b/docker/tb-transports/coap/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/coap/conf/tb-coap-transport.conf b/docker/tb-transports/coap/conf/tb-coap-transport.conf index 7695e52d44..a64a5f4828 100644 --- a/docker/tb-transports/coap/conf/tb-coap-transport.conf +++ b/docker/tb-transports/coap/conf/tb-coap-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/http/conf/logback.xml b/docker/tb-transports/http/conf/logback.xml index 0aaed3ca3b..67d15ce607 100644 --- a/docker/tb-transports/http/conf/logback.xml +++ b/docker/tb-transports/http/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/http/conf/tb-http-transport.conf b/docker/tb-transports/http/conf/tb-http-transport.conf index b52f5a0f28..67d217e6c3 100644 --- a/docker/tb-transports/http/conf/tb-http-transport.conf +++ b/docker/tb-transports/http/conf/tb-http-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/mqtt/conf/logback.xml b/docker/tb-transports/mqtt/conf/logback.xml index 4a4e19d00e..494759aeb9 100644 --- a/docker/tb-transports/mqtt/conf/logback.xml +++ b/docker/tb-transports/mqtt/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf index 4081d5a194..4fa2550acf 100644 --- a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf +++ b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/k8s/tb-coap-transport-configmap.yml b/k8s/tb-coap-transport-configmap.yml index d2ef3fbb47..55adbcbdec 100644 --- a/k8s/tb-coap-transport-configmap.yml +++ b/k8s/tb-coap-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-coap-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-http-transport-configmap.yml b/k8s/tb-http-transport-configmap.yml index f406fd0d4f..de20a90952 100644 --- a/k8s/tb-http-transport-configmap.yml +++ b/k8s/tb-http-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-http-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-mqtt-transport-configmap.yml b/k8s/tb-mqtt-transport-configmap.yml index cf9b2c655e..9d56f6fa71 100644 --- a/k8s/tb-mqtt-transport-configmap.yml +++ b/k8s/tb-mqtt-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-mqtt-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-node-configmap.yml b/k8s/tb-node-configmap.yml index 7fa9612600..1f6f1abf97 100644 --- a/k8s/tb-node-configmap.yml +++ b/k8s/tb-node-configmap.yml @@ -24,7 +24,7 @@ metadata: data: conf: | export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -37,10 +37,10 @@ data: - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/thingsboard.yml b/k8s/thingsboard.yml index 13fadf7fe2..8f50f34f8e 100644 --- a/k8s/thingsboard.yml +++ b/k8s/thingsboard.yml @@ -247,18 +247,12 @@ spec: - containerPort: 9001 name: rpc env: - - name: RPC_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: ZOOKEEPER_ENABLED value: "true" - name: ZOOKEEPER_URL @@ -334,14 +328,12 @@ spec: - containerPort: 1883 name: mqtt env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: MQTT_BIND_ADDRESS value: "0.0.0.0" - name: MQTT_BIND_PORT @@ -409,14 +401,12 @@ spec: - containerPort: 8080 name: http env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: HTTP_BIND_ADDRESS value: "0.0.0.0" - name: HTTP_BIND_PORT @@ -484,14 +474,12 @@ spec: name: coap protocol: UDP env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: COAP_BIND_ADDRESS value: "0.0.0.0" - name: COAP_BIND_PORT diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..8ff890e476 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -42,7 +42,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..4cd67df020 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -43,7 +43,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..b4af9547c8 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -73,7 +73,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" From b5bedbfdf6e99ee133bad22011af4299a8d38854 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 14:54:25 +0300 Subject: [PATCH 32/39] Refactoring of Docker Compose --- .../demo/rule_chains/root_rule_chain.json | 2 +- .../controller/RuleChainController.java | 14 ++--- .../server/dao/rule/BaseRuleChainService.java | 25 +++++--- docker/README.md | 2 +- docker/docker-compose.cassandra.yml | 18 +++++- docker/docker-compose.postgres.volumes.yml | 10 +++- docker/docker-compose.postgres.yml | 18 +++++- docker/docker-compose.yml | 58 +++++++++++++++++-- docker/docker-install-tb.sh | 2 +- docker/docker-upgrade-tb.sh | 4 +- docker/haproxy/config/haproxy.cfg | 4 +- docker/tb-js-executor.env | 2 +- docker/tb-node.env | 2 +- .../server/msa/ThingsBoardDbInstaller.java | 2 +- 14 files changed, 125 insertions(+), 38 deletions(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 1b321b1e1e..b68d070861 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -3,7 +3,7 @@ "additionalInfo": null, "name": "Root Rule Chain", "firstRuleNodeId": null, - "root": false, + "root": true, "debugMode": false, "configuration": null }, diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 5685947c9a..0bc518e48c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -161,15 +161,15 @@ public class RuleChainController extends BaseController { TenantId tenantId = getCurrentUser().getTenantId(); RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId); if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) { + if (previousRootRuleChain != null) { + previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - - tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), - ComponentLifecycleEvent.UPDATED); - - logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, - null, ActionType.UPDATED, null); + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + ComponentLifecycleEvent.UPDATED); + logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, + null, ActionType.UPDATED, null); + } ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 8ed968a78a..fdd99f97f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -88,26 +88,33 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId()); if (!ruleChain.isRoot()) { RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId()); - if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { - try { + try { + if (previousRootRuleChain == null) { + setRootAndSave(tenantId, ruleChain); + return true; + } else if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); previousRootRuleChain.setRoot(false); ruleChainDao.save(tenantId, previousRootRuleChain); - createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); - ruleChain.setRoot(true); - ruleChainDao.save(tenantId, ruleChain); + setRootAndSave(tenantId, ruleChain); return true; - } catch (ExecutionException | InterruptedException e) { - log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); - throw new RuntimeException(e); } + } catch (ExecutionException | InterruptedException e) { + log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); + throw new RuntimeException(e); } } return false; } + private void setRootAndSave(TenantId tenantId, RuleChain ruleChain) throws ExecutionException, InterruptedException { + createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); + ruleChain.setRoot(true); + ruleChainDao.save(tenantId, ruleChain); + } + @Override public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) { Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id."); diff --git a/docker/README.md b/docker/README.md index ff61c2599b..9436386010 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ In case of any issues you can examine service logs for errors. For example to see ThingsBoard node logs execute the following command: ` -$ docker-compose logs -f tb1 +$ docker-compose logs -f tb-core1 tb-rule-engine1 ` Or use `docker-compose ps` to see the state of all the containers. diff --git a/docker/docker-compose.cassandra.yml b/docker/docker-compose.cassandra.yml index 49de5f396c..1066570140 100644 --- a/docker/docker-compose.cassandra.yml +++ b/docker/docker-compose.cassandra.yml @@ -24,14 +24,28 @@ services: - "9042" volumes: - ./tb-node/cassandra:/var/lib/cassandra - tb1: + tb-core1: env_file: - tb-node.cassandra.env depends_on: - kafka - redis - cassandra - tb2: + tb-core2: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine1: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine2: env_file: - tb-node.cassandra.env depends_on: diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 1331393044..caaacf96e8 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,10 +20,16 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb1: + tb-core1: volumes: - tb-log-volume:/var/log/thingsboard - tb2: + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: volumes: - tb-log-volume:/var/log/thingsboard tb-coap-transport: diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index d47cbd6c37..3d19e5f968 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -27,14 +27,28 @@ services: POSTGRES_PASSWORD: postgres volumes: - ./tb-node/postgres:/var/lib/postgresql/data - tb1: + tb-core1: env_file: - tb-node.postgres.env depends_on: - kafka - redis - postgres - tb2: + tb-core2: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine1: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9774346b3d..783a5ade46 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,7 +48,7 @@ services: - tb-js-executor.env depends_on: - kafka - tb1: + tb-core1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -59,7 +59,8 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb1 + TB_SERVICE_ID: tb-core1 + TB_SERVICE_TYPE: tb-core env_file: - tb-node.env volumes: @@ -69,7 +70,7 @@ services: - kafka - redis - tb-js-executor - tb2: + tb-core2: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -80,7 +81,52 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb2 + TB_SERVICE_ID: tb-core2 + TB_SERVICE_TYPE: tb-core + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine1: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine1 + TB_SERVICE_TYPE: tb-rule-engine + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine2: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine2 + TB_SERVICE_TYPE: tb-rule-engine env_file: - tb-node.env volumes: @@ -195,8 +241,8 @@ services: MQTT_PORT: 1883 FORCE_HTTPS_REDIRECT: "false" links: - - tb1 - - tb2 + - tb-core1 + - tb-core2 - tb-web-ui1 - tb-web-ui2 - tb-mqtt-transport1 diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 1a5d75d64a..cabb3c190d 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,6 +49,6 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 7db7ff6172..e62dbdc438 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,8 +44,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/haproxy/config/haproxy.cfg b/docker/haproxy/config/haproxy.cfg index ba17bd64d9..bc36364024 100644 --- a/docker/haproxy/config/haproxy.cfg +++ b/docker/haproxy/config/haproxy.cfg @@ -111,6 +111,6 @@ backend tb-api-backend balance leastconn option tcp-check option log-health-checks - server tbApi1 tb1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 - server tbApi2 tb2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi1 tb-core1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi2 tb-core2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 http-request set-header X-Forwarded-Port %[dst_port] diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 38f283cd1a..0b64f43b00 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,5 +1,5 @@ -REMOTE_JS_EVAL_REQUEST_TOPIC=js.eval.requests +REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info LOG_FOLDER=logs diff --git a/docker/tb-node.env b/docker/tb-node.env index aec55e9205..542296babe 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,7 +4,7 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=local +JS_EVALUATOR=remote TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 8fa7ae9a3d..d661bb7476 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -92,7 +92,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); dockerCompose.invokeCompose(); } finally { From a779839081b92a995afa67427b53bdfefe34dc92 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 19:16:25 +0300 Subject: [PATCH 33/39] Fixed transport --- .../org/thingsboard/server/actors/ActorSystemContext.java | 3 +++ .../common/transport/service/DefaultTransportService.java | 6 ++++++ docker/tb-coap-transport.env | 2 ++ docker/tb-http-transport.env | 2 ++ docker/tb-mqtt-transport.env | 2 ++ 5 files changed, 15 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e9b68882b..28f5b188e5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,6 +32,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -233,6 +234,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-core mode */ + @Lazy @Autowired(required = false) @Getter private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; @@ -240,6 +242,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-rule-engine mode */ + @Lazy @Autowired(required = false) @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index c820167de1..3db7300bc3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -544,6 +544,9 @@ public class DefaultTransportService implements TransportService { protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg); + } tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? @@ -552,6 +555,9 @@ public class DefaultTransportService implements TransportService { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg); + } ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index af95bc904d..8a75904185 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 71bc434241..e62424290f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 163cfab451..54c35f355c 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 From 4aa225babaced85e28bdf72917de1ed437b88bc7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Apr 2020 10:54:59 +0300 Subject: [PATCH 34/39] added exclusions to azure dependency, and added logger level OFF to CoreMessageReceiver --- application/src/main/resources/logback.xml | 3 +++ pom.xml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 614f85ba0e..e25fb72ccb 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -31,8 +31,11 @@ + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8876e68538..845d590faf 100755 --- a/pom.xml +++ b/pom.xml @@ -907,6 +907,12 @@ com.microsoft.azure azure-servicebus ${azure-servicebus.version} + + + com.microsoft.azure + adal4j + + org.passay From 5b8122e1b3b4fbfcd90ab063ae0231cf12dee98f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Apr 2020 15:52:36 +0300 Subject: [PATCH 35/39] added --- application/src/main/conf/logback.xml | 1 + msa/tb/docker/logback.xml | 1 + transport/coap/src/main/conf/logback.xml | 2 ++ transport/coap/src/main/resources/logback.xml | 2 ++ transport/http/src/main/conf/logback.xml | 2 ++ transport/http/src/main/resources/logback.xml | 2 ++ transport/mqtt/src/main/conf/logback.xml | 2 ++ transport/mqtt/src/main/resources/logback.xml | 2 ++ 8 files changed, 14 insertions(+) diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 295073bb82..46b42182fb 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,7 @@ + diff --git a/msa/tb/docker/logback.xml b/msa/tb/docker/logback.xml index 780a2add2f..256e78436f 100644 --- a/msa/tb/docker/logback.xml +++ b/msa/tb/docker/logback.xml @@ -42,6 +42,7 @@ + diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + From 30e5af36274b5e20655044d38e7b0bcbc98093ea Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 24 Apr 2020 18:55:11 +0300 Subject: [PATCH 36/39] added zookeeper configs to coap and http --- .../src/main/resources/tb-coap-transport.yml | 20 ++++++++++++++----- .../src/main/resources/tb-http-transport.yml | 20 ++++++++++++++----- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..2d70330187 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # COAP server parameters transport: @@ -42,7 +52,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..0b86dc026f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -20,10 +20,20 @@ server: # Server bind port port: "${HTTP_BIND_PORT:8081}" -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # HTTP server parameters transport: @@ -43,7 +53,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..e267016623 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -73,7 +73,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" From ba140644c7b95f18295c5836fc6b1fe95b904ed9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 26 Apr 2020 00:57:46 +0300 Subject: [PATCH 37/39] added Azure Service Bus queue settings --- .../src/main/resources/thingsboard.yml | 6 ++ .../azure/servicebus/TbServiceBusAdmin.java | 38 ++++++++-- .../servicebus/TbServiceBusQueueConfigs.java | 71 +++++++++++++++++++ .../provider/KafkaMonolithQueueFactory.java | 20 ++++++ .../provider/KafkaTbCoreQueueFactory.java | 19 +++++ .../KafkaTbRuleEngineQueueFactory.java | 17 +++++ .../KafkaTbTransportQueueFactory.java | 18 +++++ .../ServiceBusMonolithQueueFactory.java | 63 ++++++++++++---- .../ServiceBusTbCoreQueueFactory.java | 57 +++++++++++---- .../ServiceBusTbRuleEngineQueueFactory.java | 48 ++++++++++--- .../ServiceBusTransportQueueFactory.java | 47 ++++++++---- .../src/main/resources/tb-coap-transport.yml | 6 ++ .../src/main/resources/tb-http-transport.yml | 6 ++ .../src/main/resources/tb-mqtt-transport.yml | 6 ++ 14 files changed, 368 insertions(+), 54 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3df3cb9559..efd076f6ea 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -556,6 +556,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index 336514b7b2..229d6b4244 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -16,27 +16,32 @@ package org.thingsboard.server.queue.azure.servicebus; import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.management.QueueDescription; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.time.Duration; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") public class TbServiceBusAdmin implements TbQueueAdmin { + private final String MAX_SIZE = "maxSizeInMb"; + private final String MESSAGE_TIME_TO_LIVE = "messageTimeToLiveInSec"; + private final String LOCK_DURATION = "lockDurationInSec"; + private final Map queueConfigs; private final Set queues = ConcurrentHashMap.newKeySet(); private final ManagementClient client; - public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings, Map queueConfigs) { + this.queueConfigs = queueConfigs; + ConnectionStringBuilder builder = new ConnectionStringBuilder( serviceBusSettings.getNamespaceName(), "queues", @@ -60,13 +65,34 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } try { - client.createQueue(topic); + QueueDescription queueDescription = new QueueDescription(topic); + setQueueConfigs(queueDescription); + + client.createQueue(queueDescription); queues.add(topic); } catch (ServiceBusException | InterruptedException e) { log.error("Failed to create queue: [{}]", topic, e); } } + private void setQueueConfigs(QueueDescription queueDescription) { + queueConfigs.forEach((confKey, confValue) -> { + switch (confKey) { + case MAX_SIZE: + queueDescription.setMaxSizeInMB(Long.parseLong(confValue)); + break; + case MESSAGE_TIME_TO_LIVE: + queueDescription.setDefaultMessageTimeToLive(Duration.ofSeconds(Long.parseLong(confValue))); + break; + case LOCK_DURATION: + queueDescription.setLockDuration(Duration.ofSeconds(Long.parseLong(confValue))); + break; + default: + log.error("Unknown config: [{}]", confKey); + } + }); + } + public void destroy() { try { client.close(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java new file mode 100644 index 0000000000..21ab455d6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusQueueConfigs { + @Value("${queue.service-bus.queue-properties.core}") + private String coreProperties; + @Value("${queue.service-bus.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.service-bus.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.service-bus.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.service-bus.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 72c6f7b8b9..2aca5e6da1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -49,6 +49,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -247,4 +248,23 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); return builder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index ca10830981..632a6e480e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -47,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -218,4 +219,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index bcac399a64..dc44caf3f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -45,6 +45,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -190,4 +191,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); return builder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 2f7aab4779..505aa203d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -135,4 +137,20 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.admin(notificationAdmin); return responseBuilder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 8b01c38607..3c82e18d91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -30,19 +30,25 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -54,7 +60,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -62,7 +73,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -70,73 +81,97 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 523ac88bd1..bcb48630f9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -29,8 +29,10 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -40,6 +42,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -50,7 +54,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, TbQueueCoreSettings coreSettings, @@ -58,67 +67,91 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 8a2f9b396b..1f492596a1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -38,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -47,55 +51,63 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @@ -104,4 +116,20 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index 33c5277ca5..0b5640d384 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -23,6 +23,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -33,6 +35,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -40,37 +44,43 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbQueueProducer> producerTemplate = - new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); TbQueueConsumer> consumerTemplate = - new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,18 +91,31 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 2d70330187..47a011c35f 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -95,6 +95,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0b86dc026f..55236dd84f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -96,6 +96,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e267016623..3872a6d0b3 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -116,6 +116,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" From 86f21023febedb362e8f991a97886d5cee295dec Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Apr 2020 14:44:29 +0300 Subject: [PATCH 38/39] Improved PartitionService and fixed startup order in docker-compose.yml --- .../ConsistentHashParitionServiceTest.java | 21 +++++++++++++------ .../ConsistentHashPartitionService.java | 12 ++++++++++- docker/docker-compose.yml | 4 ++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 3f7619ed1e..6b03482c0e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cluster.routing; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,6 +32,8 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import java.util.ArrayList; import java.util.Collections; @@ -41,6 +44,7 @@ import java.util.Map; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -52,6 +56,7 @@ public class ConsistentHashParitionServiceTest { private TbServiceInfoProvider discoveryService; private TenantRoutingInfoService routingInfoService; private ApplicationEventPublisher applicationEventPublisher; + private TbQueueRuleEngineSettings ruleEngineSettings; private String hashFunctionName = "murmur3_128"; private Integer virtualNodesSize = 16; @@ -62,12 +67,15 @@ public class ConsistentHashParitionServiceTest { discoveryService = mock(TbServiceInfoProvider.class); applicationEventPublisher = mock(ApplicationEventPublisher.class); routingInfoService = mock(TenantRoutingInfoService.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); + ruleEngineSettings = mock(TbQueueRuleEngineSettings.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, + routingInfoService, + applicationEventPublisher, + ruleEngineSettings + ); + when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList()); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() @@ -107,8 +115,9 @@ public class ConsistentHashParitionServiceTest { List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); - + double diffPercent = (diff / ITERATIONS) * 100.0; + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)"); + Assert.assertTrue(diffPercent < 0.5); for (Map.Entry entry : data) { System.out.println(entry.getKey() + ": " + entry.getValue()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index df25ea3ba9..918570bc56 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; @@ -61,6 +62,7 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; + private final TbQueueRuleEngineSettings tbQueueRuleEngineSettings; private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); @@ -74,10 +76,14 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, + TenantRoutingInfoService tenantRoutingInfoService, + ApplicationEventPublisher applicationEventPublisher, + TbQueueRuleEngineSettings tbQueueRuleEngineSettings) { this.serviceInfoProvider = serviceInfoProvider; this.tenantRoutingInfoService = tenantRoutingInfoService; this.applicationEventPublisher = applicationEventPublisher; + this.tbQueueRuleEngineSettings = tbQueueRuleEngineSettings; } @PostConstruct @@ -85,6 +91,10 @@ public class ConsistentHashPartitionService implements PartitionService { this.hashFunction = forName(hashFunctionName); partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); + tbQueueRuleEngineSettings.getQueues().forEach(queueConfiguration -> { + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getTopic()); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getPartitions()); + }); } @Override diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 783a5ade46..2f0d0fb3b3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -70,6 +70,8 @@ services: - kafka - redis - tb-js-executor + - tb-rule-engine1 + - tb-rule-engine2 tb-core2: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" @@ -92,6 +94,8 @@ services: - kafka - redis - tb-js-executor + - tb-rule-engine1 + - tb-rule-engine2 tb-rule-engine1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" From cb6e91277f8dad93a2f09d2f1d3efce35e9cd9e0 Mon Sep 17 00:00:00 2001 From: bdrlamb Date: Mon, 27 Apr 2020 14:32:35 -0700 Subject: [PATCH 39/39] Fix MQTT inactivity disconnects MQTT clients don't send PINGREQ messages if they've recently sent [UN]SUBSCRIBE/PUBLISH messages. The MQTT server last activity timestamp is currently only updated in response to PINGREQ messages. This causes the server to disconnect any clients that regularly send PUBLISH messages since they don't send PINGREQ. This change adds reportActivity calls in response to PUBLISH, SUBSCRIBE and UNSUBSCRIBE in addition to PINGREQ. --- .../server/transport/mqtt/MqttTransportHandler.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 7295cb786b..d0f9c489cc 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -138,12 +138,24 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement break; case PUBLISH: processPublish(ctx, (MqttPublishMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case SUBSCRIBE: processSubscribe(ctx, (MqttSubscribeMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case UNSUBSCRIBE: processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case PINGREQ: if (checkConnected(ctx, msg)) {